diff --git "a/logs/20250527_235509/train.log" "b/logs/20250527_235509/train.log"
new file mode 100644--- /dev/null
+++ "b/logs/20250527_235509/train.log"
@@ -0,0 +1,4035 @@
+2025-05-27 23:55:29,745	INFO dashboard_sdk.py:338 -- Uploading package gcs://_ray_pkg_e25cfeb4a729bbcc.zip.
+2025-05-27 23:55:29,746	INFO packaging.py:575 -- Creating a file package for local module '/mnt/petrelfs/luyiting/MultiAgentEval/lmm-r1'.
+2025-05-27 23:55:28,765	INFO cli.py:39 -- Job submission server address: http://127.0.0.1:2989
+2025-05-27 23:55:34,243	SUCC cli.py:63 -- -------------------------------------------------------
+2025-05-27 23:55:34,243	SUCC cli.py:64 -- Job 'raysubmit_tdQtQ5pUh6YdmvqQ' submitted successfully
+2025-05-27 23:55:34,243	SUCC cli.py:65 -- -------------------------------------------------------
+2025-05-27 23:55:34,243	INFO cli.py:289 -- Next steps
+2025-05-27 23:55:34,243	INFO cli.py:290 -- Query the logs of the job:
+2025-05-27 23:55:34,243	INFO cli.py:292 -- ray job logs raysubmit_tdQtQ5pUh6YdmvqQ
+2025-05-27 23:55:34,243	INFO cli.py:294 -- Query the status of the job:
+2025-05-27 23:55:34,243	INFO cli.py:296 -- ray job status raysubmit_tdQtQ5pUh6YdmvqQ
+2025-05-27 23:55:34,244	INFO cli.py:298 -- Request the job to be stopped:
+2025-05-27 23:55:34,244	INFO cli.py:300 -- ray job stop raysubmit_tdQtQ5pUh6YdmvqQ
+2025-05-27 23:55:34,246	INFO cli.py:307 -- Tailing logs until the job exits (disable with --no-wait):
+2025-05-27 23:55:33,741	INFO job_manager.py:531 -- Runtime env is setting up.
+[2025-05-27 23:55:52,909] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+INFO 05-27 23:55:57 [__init__.py:239] Automatically detected platform cuda.
+2025-05-27 23:55:58,401	INFO worker.py:1520 -- Using address 10.140.0.151:6239 set in the environment variable RAY_ADDRESS
+2025-05-27 23:55:58,402	INFO worker.py:1660 -- Connecting to existing Ray cluster at address: 10.140.0.151:6239...
+2025-05-27 23:55:58,424	INFO worker.py:1843 -- Connected to Ray cluster. View the dashboard at 10.140.0.151:2989 
+(pid=425166) INFO 05-27 23:56:17 [__init__.py:239] Automatically detected platform cuda.
+(LLMRayActor pid=425166) INFO 05-27 23:56:43 [config.py:585] This model supports multiple tasks: {'classify', 'reward', 'embed', 'generate', 'score'}. Defaulting to 'generate'.
+(LLMRayActor pid=425166) WARNING 05-27 23:56:43 [arg_utils.py:1846] VLLM_ATTENTION_BACKEND=triton is not supported by the V1 Engine. Falling back to V0. We recommend to remove VLLM_ATTENTION_BACKEND=triton from your config in favor of the V1 Engine.
+(LLMRayActor pid=425166) WARNING 05-27 23:56:43 [arg_utils.py:1745] --enable-prefix-caching is not supported for multimodal models in V0 and has been disabled.
+(LLMRayActor pid=425166) INFO 05-27 23:56:43 [llm_engine.py:241] Initializing a V0 LLM engine (v0.8.2.dev76+gf68cce8) with config: model='/mnt/petrelfs/luyiting/ckt/Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct/', speculative_config=None, tokenizer='/mnt/petrelfs/luyiting/ckt/Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct/', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='xgrammar', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=42, served_model_name=/mnt/petrelfs/luyiting/ckt/Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct/, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=False, chunked_prefill_enabled=False, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"splitting_ops":[],"compile_sizes":[],"cudagraph_capture_sizes":[256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":256}, use_cached_outputs=False, 
+(pid=425165) INFO 05-27 23:56:17 [__init__.py:239] Automatically detected platform cuda. [repeated 7x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)
+(LLMRayActor pid=425160) INFO 05-27 23:56:43 [config.py:585] This model supports multiple tasks: {'classify', 'embed', 'reward', 'score', 'generate'}. Defaulting to 'generate'.
+(LLMRayActor pid=425162) INFO 05-27 23:56:43 [config.py:585] This model supports multiple tasks: {'generate', 'classify', 'score', 'reward', 'embed'}. Defaulting to 'generate'.
+(LLMRayActor pid=425161) INFO 05-27 23:56:43 [config.py:585] This model supports multiple tasks: {'reward', 'generate', 'embed', 'classify', 'score'}. Defaulting to 'generate'.
+(LLMRayActor pid=425159) INFO 05-27 23:56:43 [config.py:585] This model supports multiple tasks: {'generate', 'score', 'classify', 'embed', 'reward'}. Defaulting to 'generate'.
+(LLMRayActor pid=425163) INFO 05-27 23:56:43 [config.py:585] This model supports multiple tasks: {'embed', 'score', 'reward', 'generate', 'classify'}. Defaulting to 'generate'.
+(LLMRayActor pid=425164) INFO 05-27 23:56:43 [config.py:585] This model supports multiple tasks: {'embed', 'classify', 'score', 'reward', 'generate'}. Defaulting to 'generate'.
+(LLMRayActor pid=425165) INFO 05-27 23:56:43 [config.py:585] This model supports multiple tasks: {'generate', 'reward', 'embed', 'classify', 'score'}. Defaulting to 'generate'.
+(LLMRayActor pid=425166) [2025-05-27 23:56:46,863] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+(LLMRayActor pid=425166) INFO 05-27 23:56:51 [cuda.py:293] Using Flash Attention backend.
+(LLMRayActor pid=425165) WARNING 05-27 23:56:43 [arg_utils.py:1846] VLLM_ATTENTION_BACKEND=triton is not supported by the V1 Engine. Falling back to V0. We recommend to remove VLLM_ATTENTION_BACKEND=triton from your config in favor of the V1 Engine. [repeated 7x across cluster]
+(LLMRayActor pid=425165) WARNING 05-27 23:56:43 [arg_utils.py:1745] --enable-prefix-caching is not supported for multimodal models in V0 and has been disabled. [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-27 23:56:43 [llm_engine.py:241] Initializing a V0 LLM engine (v0.8.2.dev76+gf68cce8) with config: model='/mnt/petrelfs/luyiting/ckt/Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct/', speculative_config=None, tokenizer='/mnt/petrelfs/luyiting/ckt/Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct/', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='xgrammar', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=49, served_model_name=/mnt/petrelfs/luyiting/ckt/Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct/, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=False, chunked_prefill_enabled=False, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"splitting_ops":[],"compile_sizes":[],"cudagraph_capture_sizes":[256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":256}, use_cached_outputs=False,  [repeated 7x across cluster]
+(LLMRayActor pid=425166) INFO 05-27 23:56:54 [parallel_state.py:967] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0
+(LLMRayActor pid=425166) INFO 05-27 23:56:54 [model_runner.py:1110] Starting to load model /mnt/petrelfs/luyiting/ckt/Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct/...
+(LLMRayActor pid=425165) [2025-05-27 23:56:46,860] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect) [repeated 7x across cluster]
+(LLMRayActor pid=425166) INFO 05-27 23:56:55 [config.py:3229] cudagraph sizes specified by model runner [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256] is overridden by config [256, 128, 2, 1, 4, 136, 8, 144, 16, 152, 24, 160, 32, 168, 40, 176, 48, 184, 56, 192, 64, 200, 72, 208, 80, 216, 88, 120, 224, 96, 232, 104, 240, 112, 248]
+(LLMRayActor pid=425166) 
+Loading safetensors checkpoint shards:   0% Completed | 0/5 [00:00<?, ?it/s]
+(LLMRayActor pid=425166) 
+Loading safetensors checkpoint shards:  20% Completed | 1/5 [00:01<00:07,  1.90s/it]
+(LLMRayActor pid=425159) 
+Loading safetensors checkpoint shards:   0% Completed | 0/5 [00:00<?, ?it/s] [repeated 7x across cluster]
+(LLMRayActor pid=425166) 
+Loading safetensors checkpoint shards:  40% Completed | 2/5 [00:07<00:11,  3.93s/it] [repeated 8x across cluster]
+(LLMRayActor pid=425166) 
+Loading safetensors checkpoint shards:  60% Completed | 3/5 [00:12<00:09,  4.68s/it] [repeated 8x across cluster]
+(LLMRayActor pid=425166) 
+Loading safetensors checkpoint shards:  80% Completed | 4/5 [00:17<00:04,  4.78s/it] [repeated 8x across cluster]
+(LLMRayActor pid=425166) 
+(LLMRayActor pid=425166) 
+Loading safetensors checkpoint shards: 100% Completed | 5/5 [00:23<00:00,  4.76s/it] [repeated 9x across cluster]
+(LLMRayActor pid=425163) 
+(LLMRayActor pid=425162) 
+(LLMRayActor pid=425161) INFO 05-27 23:57:19 [loader.py:429] Loading weights took 23.93 seconds
+(LLMRayActor pid=425165) INFO 05-27 23:56:51 [cuda.py:293] Using Flash Attention backend. [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-27 23:56:54 [parallel_state.py:967] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0 [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-27 23:56:54 [model_runner.py:1110] Starting to load model /mnt/petrelfs/luyiting/ckt/Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct/... [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-27 23:56:55 [config.py:3229] cudagraph sizes specified by model runner [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256] is overridden by config [256, 128, 2, 1, 4, 136, 8, 144, 16, 152, 24, 160, 32, 168, 40, 176, 48, 184, 56, 192, 64, 200, 72, 208, 80, 216, 88, 120, 224, 96, 232, 104, 240, 112, 248] [repeated 7x across cluster]
+(LLMRayActor pid=425161) 
+(LLMRayActor pid=425160) 
+(LLMRayActor pid=425165) 
+(LLMRayActor pid=425164) 
+(LLMRayActor pid=425159) 
+(LLMRayActor pid=425161) INFO 05-27 23:57:19 [model_runner.py:1146] Model loading took 15.6271 GB and 24.389975 seconds
+(LLMRayActor pid=425161) WARNING 05-27 23:57:20 [model_runner.py:1296] Computed max_num_seqs (min(256, 8192 // 32768)) to be less than 1. Setting it to the minimum value of 1.
+(LLMRayActor pid=425161) Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+(LLMRayActor pid=425161) WARNING 05-27 23:57:25 [profiling.py:222] The sequence length used for profiling (max_num_batched_tokens / max_num_seqs = 8192) is too short to hold the multi-modal embeddings in the worst case (32768 tokens in total, out of which {'image': 16384, 'video': 16384} are reserved for multi-modal embeddings). This may cause certain multi-modal inputs to fail during inference, even when the input text is short. To avoid this, you should increase `max_model_len`, reduce `max_num_seqs`, and/or reduce `mm_counts`.
+(LLMRayActor pid=425165) INFO 05-27 23:57:19 [loader.py:429] Loading weights took 24.23 seconds [repeated 7x across cluster]
+(LLMRayActor pid=425166) INFO 05-27 23:57:19 [model_runner.py:1146] Model loading took 15.6271 GB and 24.639012 seconds [repeated 7x across cluster]
+(LLMRayActor pid=425165) WARNING 05-27 23:57:20 [model_runner.py:1296] Computed max_num_seqs (min(256, 8192 // 32768)) to be less than 1. Setting it to the minimum value of 1. [repeated 7x across cluster]
+(LLMRayActor pid=425161) INFO 05-27 23:57:27 [worker.py:267] Memory profiling takes 8.11 seconds
+(LLMRayActor pid=425161) INFO 05-27 23:57:27 [worker.py:267] the current vLLM instance can use total_gpu_memory (79.32GiB) x gpu_memory_utilization (0.50) = 39.66GiB
+(LLMRayActor pid=425161) INFO 05-27 23:57:27 [worker.py:267] model weights take 15.63GiB; non_torch_memory takes 0.21GiB; PyTorch activation peak memory takes 1.09GiB; the rest of the memory reserved for KV Cache is 22.73GiB.
+(LLMRayActor pid=425161) INFO 05-27 23:57:28 [executor_base.py:111] # cuda blocks: 26598, # CPU blocks: 4681
+(LLMRayActor pid=425161) INFO 05-27 23:57:28 [executor_base.py:116] Maximum concurrency for 8192 tokens per request: 51.95x
+(LLMRayActor pid=425161) INFO 05-27 23:57:31 [model_runner.py:1442] Capturing cudagraphs for decoding. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI. If out-of-memory error occurs during cudagraph capture, consider decreasing `gpu_memory_utilization` or switching to eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.
+(LLMRayActor pid=425162) WARNING 05-27 23:57:26 [profiling.py:222] The sequence length used for profiling (max_num_batched_tokens / max_num_seqs = 8192) is too short to hold the multi-modal embeddings in the worst case (32768 tokens in total, out of which {'image': 16384, 'video': 16384} are reserved for multi-modal embeddings). This may cause certain multi-modal inputs to fail during inference, even when the input text is short. To avoid this, you should increase `max_model_len`, reduce `max_num_seqs`, and/or reduce `mm_counts`. [repeated 7x across cluster]
+(LLMRayActor pid=425161) 
+Capturing CUDA graph shapes:   0%|          | 0/35 [00:00<?, ?it/s]
+(LLMRayActor pid=425159) 
+Loading safetensors checkpoint shards: 100% Completed | 5/5 [00:23<00:00,  4.76s/it] [repeated 14x across cluster]
+(LLMRayActor pid=425159) Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. [repeated 7x across cluster]
+(LLMRayActor pid=425163) 
+Capturing CUDA graph shapes:   3%|▎         | 1/35 [00:00<00:18,  1.87it/s]
+(LLMRayActor pid=425162) 
+Capturing CUDA graph shapes:   0%|          | 0/35 [00:00<?, ?it/s] [repeated 7x across cluster]
+(LLMRayActor pid=425166) 
+Capturing CUDA graph shapes:  31%|███▏      | 11/35 [00:05<00:12,  1.93it/s] [repeated 79x across cluster]
+(LLMRayActor pid=425163) 
+Capturing CUDA graph shapes:  57%|█████▋    | 20/35 [00:10<00:08,  1.71it/s] [repeated 80x across cluster]
+(LLMRayActor pid=425166) 
+Capturing CUDA graph shapes:  86%|████████▌ | 30/35 [00:15<00:02,  1.94it/s] [repeated 74x across cluster]
+(LLMRayActor pid=425160) 
+Capturing CUDA graph shapes:  91%|█████████▏| 32/35 [00:16<00:01,  1.98it/s]
+(LLMRayActor pid=425160) INFO 05-27 23:57:49 [model_runner.py:1570] Graph capturing finished in 18 secs, took 1.91 GiB
+(LLMRayActor pid=425160) INFO 05-27 23:57:49 [llm_engine.py:447] init engine (profile, create kv cache, warmup model) took 29.32 seconds
+(LLMRayActor pid=425162) INFO 05-27 23:57:28 [worker.py:267] Memory profiling takes 8.37 seconds [repeated 7x across cluster]
+(LLMRayActor pid=425162) INFO 05-27 23:57:28 [worker.py:267] the current vLLM instance can use total_gpu_memory (79.32GiB) x gpu_memory_utilization (0.50) = 39.66GiB [repeated 7x across cluster]
+(LLMRayActor pid=425162) INFO 05-27 23:57:28 [worker.py:267] model weights take 15.63GiB; non_torch_memory takes 0.21GiB; PyTorch activation peak memory takes 1.09GiB; the rest of the memory reserved for KV Cache is 22.73GiB. [repeated 7x across cluster]
+(LLMRayActor pid=425162) INFO 05-27 23:57:28 [executor_base.py:111] # cuda blocks: 26598, # CPU blocks: 4681 [repeated 7x across cluster]
+(LLMRayActor pid=425162) INFO 05-27 23:57:28 [executor_base.py:116] Maximum concurrency for 8192 tokens per request: 51.95x [repeated 7x across cluster]
+(LLMRayActor pid=425162) INFO 05-27 23:57:31 [model_runner.py:1442] Capturing cudagraphs for decoding. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI. If out-of-memory error occurs during cudagraph capture, consider decreasing `gpu_memory_utilization` or switching to eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage. [repeated 7x across cluster]
+(LLMRayActor pid=425160) 
+Capturing CUDA graph shapes: 100%|██████████| 35/35 [00:17<00:00,  1.97it/s]
+Capturing CUDA graph shapes: 100%|██████████| 35/35 [00:17<00:00,  1.98it/s]
+(LLMRayActor pid=425160) INFO 05-27 23:58:00 [worker.py:133] Sleep mode freed 38.48 GiB memory, 3.87 GiB memory is still in use.
+(LLMRayActor pid=425160) INFO 05-27 23:58:00 [executor_base.py:208] It took 10.859559 seconds to fall asleep.
+(LLMRayActor pid=425162) INFO 05-27 23:57:52 [model_runner.py:1570] Graph capturing finished in 20 secs, took 1.91 GiB [repeated 7x across cluster]
+(LLMRayActor pid=425162) INFO 05-27 23:57:52 [llm_engine.py:447] init engine (profile, create kv cache, warmup model) took 32.36 seconds [repeated 7x across cluster]
+(pid=430510) [2025-05-27 23:58:18,999] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+(LLMRayActor pid=425162) INFO 05-27 23:58:03 [worker.py:133] Sleep mode freed 38.48 GiB memory, 3.87 GiB memory is still in use. [repeated 7x across cluster]
+(LLMRayActor pid=425162) INFO 05-27 23:58:03 [executor_base.py:208] It took 10.432220 seconds to fall asleep. [repeated 7x across cluster]
+(pid=430510) INFO 05-27 23:58:22 [__init__.py:239] Automatically detected platform cuda.
+(pid=431336) [2025-05-27 23:58:41,887] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+(pid=431333) [2025-05-27 23:58:41,887] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+(pid=431336) INFO 05-27 23:58:46 [__init__.py:239] Automatically detected platform cuda.
+(ActorModelRayActor pid=430510) [2025-05-27 23:58:48,211] [INFO] [comm.py:658:init_distributed] cdb=None
+(ActorModelRayActor pid=430510) [2025-05-27 23:58:48,211] [INFO] [comm.py:689:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+(pid=431335) [2025-05-27 23:58:41,889] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect) [repeated 6x across cluster]
+(ActorModelRayActor pid=431329) You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+(LLMRayActor pid=425162) 
+Capturing CUDA graph shapes:  89%|████████▊ | 31/35 [00:17<00:02,  1.94it/s] [repeated 14x across cluster]
+(LLMRayActor pid=425162) 
+Capturing CUDA graph shapes:  97%|█████████▋| 34/35 [00:19<00:00,  1.42it/s] [repeated 23x across cluster]
+(LLMRayActor pid=425162) 
+Capturing CUDA graph shapes: 100%|██████████| 35/35 [00:20<00:00,  1.52it/s]
+Capturing CUDA graph shapes: 100%|██████████| 35/35 [00:20<00:00,  1.71it/s] [repeated 7x across cluster]
+(ActorModelRayActor pid=431336) [2025-05-27 23:58:48,279] [INFO] [config.py:734:__init__] Config mesh_device None world_size = 8
+(ActorModelRayActor pid=430510) [2025-05-27 23:58:53,042] [INFO] [partition_parameters.py:348:__exit__] finished initializing model - num_params = 729, num_elems = 8.29B
+(pid=431335) INFO 05-27 23:58:46 [__init__.py:239] Automatically detected platform cuda. [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) 
+Loading checkpoint shards:   0%|          | 0/5 [00:00<?, ?it/s]
+(ActorModelRayActor pid=431329) 
+Loading checkpoint shards:  20%|██        | 1/5 [00:06<00:27,  6.86s/it]
+(ActorModelRayActor pid=431333) You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. [repeated 7x across cluster]
+(ActorModelRayActor pid=431328) 
+Loading checkpoint shards:   0%|          | 0/5 [00:00<?, ?it/s] [repeated 7x across cluster]
+(ActorModelRayActor pid=431329) 
+Loading checkpoint shards:  40%|████      | 2/5 [00:12<00:18,  6.15s/it] [repeated 8x across cluster]
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:58:48,204] [INFO] [comm.py:658:init_distributed] cdb=None [repeated 8x across cluster]
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:58:48,205] [INFO] [comm.py:689:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+(pid=432926) [2025-05-27 23:59:07,191] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+(ActorModelRayActor pid=431333) [2025-05-27 23:58:48,803] [INFO] [config.py:734:__init__] Config mesh_device None world_size = 8 [repeated 7x across cluster]
+(pid=432929) [2025-05-27 23:59:07,192] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+(ActorModelRayActor pid=431328) 
+Loading checkpoint shards:  60%|██████    | 3/5 [00:17<00:11,  5.75s/it] [repeated 8x across cluster]
+(pid=432926) INFO 05-27 23:59:12 [__init__.py:239] Automatically detected platform cuda.
+(pid=432927) [2025-05-27 23:59:07,193] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect) [repeated 5x across cluster]
+(ReferenceModelRayActor pid=432926) [2025-05-27 23:59:13,706] [INFO] [comm.py:658:init_distributed] cdb=None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:13,759] [INFO] [config.py:734:__init__] Config mesh_device None world_size = 8
+(ReferenceModelRayActor pid=431335) You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+(ActorModelRayActor pid=431329) 
+Loading checkpoint shards:  80%|████████  | 4/5 [00:23<00:05,  5.91s/it] [repeated 8x across cluster]
+(ActorModelRayActor pid=431329) 
+Loading checkpoint shards: 100%|██████████| 5/5 [00:24<00:00,  3.92s/it]
+Loading checkpoint shards: 100%|██████████| 5/5 [00:24<00:00,  4.87s/it]
+(ActorModelRayActor pid=431336) INFO 05-27 23:59:17 visual_utils.py:42] set VIDEO_TOTAL_PIXELS: 90316800
+(pid=432927) INFO 05-27 23:59:12 [__init__.py:239] Automatically detected platform cuda. [repeated 6x across cluster]
+(ActorModelRayActor pid=430510) Actor(
+(ActorModelRayActor pid=430510)   (model): Qwen2_5_VLForConditionalGeneration(
+(ActorModelRayActor pid=430510)     (visual): Qwen2_5_VisionTransformerPretrainedModel(
+(ActorModelRayActor pid=430510)       (patch_embed): Qwen2_5_VisionPatchEmbed(
+(ActorModelRayActor pid=430510)         (proj): Conv3d(3, 1280, kernel_size=(2, 14, 14), stride=(2, 14, 14), bias=False)
+(ActorModelRayActor pid=430510)       )
+(ActorModelRayActor pid=430510)       (rotary_pos_emb): Qwen2_5_VisionRotaryEmbedding()
+(ActorModelRayActor pid=430510)       (blocks): ModuleList(
+(ActorModelRayActor pid=430510)         (0-31): 32 x Qwen2_5_VLVisionBlock(
+(ActorModelRayActor pid=430510)           (norm1): Qwen2RMSNorm((0,), eps=1e-06)
+(ActorModelRayActor pid=430510)           (norm2): Qwen2RMSNorm((0,), eps=1e-06)
+(ActorModelRayActor pid=430510)           (attn): Qwen2_5_VLVisionFlashAttention2(
+(ActorModelRayActor pid=430510)             (qkv): Linear(in_features=1280, out_features=3840, bias=True)
+(ActorModelRayActor pid=430510)             (proj): Linear(in_features=1280, out_features=1280, bias=True)
+(ActorModelRayActor pid=430510)           )
+(ActorModelRayActor pid=430510)           (mlp): Qwen2_5_VLMLP(
+(ActorModelRayActor pid=430510)             (gate_proj): Linear(in_features=1280, out_features=3420, bias=True)
+(ActorModelRayActor pid=430510)             (up_proj): Linear(in_features=1280, out_features=3420, bias=True)
+(ActorModelRayActor pid=430510)             (down_proj): Linear(in_features=3420, out_features=1280, bias=True)
+(ActorModelRayActor pid=430510)             (act_fn): SiLU()
+(ActorModelRayActor pid=430510)           )
+(ActorModelRayActor pid=430510)         )
+(ActorModelRayActor pid=430510)       )
+(ActorModelRayActor pid=430510)       (merger): Qwen2_5_VLPatchMerger(
+(ActorModelRayActor pid=430510)         (ln_q): Qwen2RMSNorm((0,), eps=1e-06)
+(ActorModelRayActor pid=430510)         (mlp): Sequential(
+(ActorModelRayActor pid=430510)           (0): Linear(in_features=5120, out_features=5120, bias=True)
+(ActorModelRayActor pid=430510)           (1): GELU(approximate='none')
+(ActorModelRayActor pid=430510)           (2): Linear(in_features=5120, out_features=3584, bias=True)
+(ActorModelRayActor pid=430510)         )
+(ActorModelRayActor pid=430510)       )
+(ActorModelRayActor pid=430510)     )
+(ActorModelRayActor pid=430510)     (model): Qwen2_5_VLModel(
+(ActorModelRayActor pid=430510)       (embed_tokens): Embedding(152064, 3584)
+(ActorModelRayActor pid=430510)       (layers): ModuleList(
+(ActorModelRayActor pid=430510)         (0-27): 28 x Qwen2_5_VLDecoderLayer(
+(ActorModelRayActor pid=430510)           (self_attn): Qwen2_5_VLFlashAttention2(
+(ActorModelRayActor pid=430510)             (q_proj): Linear(in_features=3584, out_features=3584, bias=True)
+(ActorModelRayActor pid=430510)             (k_proj): Linear(in_features=3584, out_features=512, bias=True)
+(ActorModelRayActor pid=430510)             (v_proj): Linear(in_features=3584, out_features=512, bias=True)
+(ActorModelRayActor pid=430510)             (o_proj): Linear(in_features=3584, out_features=3584, bias=False)
+(ActorModelRayActor pid=430510)             (rotary_emb): Qwen2_5_VLRotaryEmbedding()
+(ActorModelRayActor pid=430510)           )
+(ActorModelRayActor pid=430510)           (mlp): Qwen2MLP(
+(ActorModelRayActor pid=430510)             (gate_proj): Linear(in_features=3584, out_features=18944, bias=False)
+(ActorModelRayActor pid=430510)             (up_proj): Linear(in_features=3584, out_features=18944, bias=False)
+(ActorModelRayActor pid=430510)             (down_proj): Linear(in_features=18944, out_features=3584, bias=False)
+(ActorModelRayActor pid=430510)             (act_fn): SiLU()
+(ActorModelRayActor pid=430510)           )
+(ActorModelRayActor pid=430510)           (input_layernorm): Qwen2RMSNorm((0,), eps=1e-06)
+(ActorModelRayActor pid=430510)           (post_attention_layernorm): Qwen2RMSNorm((0,), eps=1e-06)
+(ActorModelRayActor pid=430510)         )
+(ActorModelRayActor pid=430510)       )
+(ActorModelRayActor pid=430510)       (norm): Qwen2RMSNorm((0,), eps=1e-06)
+(ActorModelRayActor pid=430510)       (rotary_emb): Qwen2_5_VLRotaryEmbedding()
+(ActorModelRayActor pid=430510)     )
+(ActorModelRayActor pid=430510)     (lm_head): Linear(in_features=3584, out_features=152064, bias=False)
+(ActorModelRayActor pid=430510)   )
+(ActorModelRayActor pid=430510) )
+(ReferenceModelRayActor pid=432927) [2025-05-27 23:59:13,705] [INFO] [comm.py:658:init_distributed] cdb=None [repeated 6x across cluster]
+(ReferenceModelRayActor pid=432927) [2025-05-27 23:59:13,753] [INFO] [config.py:734:__init__] Config mesh_device None world_size = 8 [repeated 7x across cluster]
+(ReferenceModelRayActor pid=432927) You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. [repeated 7x across cluster]
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:20,795] [INFO] [partition_parameters.py:348:__exit__] finished initializing model - num_params = 729, num_elems = 8.29B
+(ReferenceModelRayActor pid=431335) 
+Loading checkpoint shards:   0%|          | 0/5 [00:00<?, ?it/s]
+(ActorModelRayActor pid=431334) Installed CUDA version 12.1 does not match the version torch was compiled with 12.4 but since the APIs are compatible, accepting this combination
+(ActorModelRayActor pid=431334) Using /mnt/petrelfs/luyiting/.cache/torch_extensions/py310_cu124 as PyTorch extensions root...
+(ActorModelRayActor pid=431329) Loading extension module cpu_adam...
+(ActorModelRayActor pid=431333) Time to load cpu_adam op: 2.141021251678467 seconds
+(ActorModelRayActor pid=431333) Adam Optimizer #0 is created with AVX512 arithmetic capability.
+(ActorModelRayActor pid=431333) Config: alpha=0.000001, betas=(0.900000, 0.950000), weight_decay=0.000000, adam_w=1
+(ActorModelRayActor pid=431330) Emitting ninja build file /mnt/petrelfs/luyiting/.cache/torch_extensions/py310_cu124/cpu_adam/build.ninja...
+(ActorModelRayActor pid=431330) Building extension module cpu_adam...
+(ActorModelRayActor pid=431330) Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+(ActorModelRayActor pid=431334) ninja: no work to do.
+(ActorModelRayActor pid=430510) 
+Loading checkpoint shards:  80%|████████  | 4/5 [00:24<00:06,  6.09s/it] [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) dataset: /mnt/petrelfs/luyiting/MultiAgentEval/data_process_v1/train_ava_mini_evalmuse_koniq_llavastyle_openrlhf_merged.jsonl
+(ActorModelRayActor pid=430510) INFO 05-27 23:59:18 visual_utils.py:42] set VIDEO_TOTAL_PIXELS: 90316800 [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) 
+Loading checkpoint shards: 100%|██████████| 5/5 [00:25<00:00,  4.29s/it]
+Loading checkpoint shards: 100%|██████████| 5/5 [00:25<00:00,  5.14s/it] [repeated 7x across cluster]
+(ReferenceModelRayActor pid=432927) 
+Loading checkpoint shards:   0%|          | 0/5 [00:00<?, ?it/s] [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) Using /mnt/petrelfs/luyiting/.cache/torch_extensions/py310_cu124 as PyTorch extensions root... [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) Loading extension module cpu_adam... [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) Emitting ninja build file /mnt/petrelfs/luyiting/.cache/torch_extensions/py310_cu124/cpu_adam/build.ninja... [repeated 3x across cluster]
+(ActorModelRayActor pid=430510) Building extension module cpu_adam... [repeated 3x across cluster]
+(ActorModelRayActor pid=430510) Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) [repeated 3x across cluster]
+(ReferenceModelRayActor pid=432924) 
+Loading checkpoint shards:  40%|████      | 2/5 [00:11<00:17,  5.75s/it] [repeated 9x across cluster]
+(ReferenceModelRayActor pid=431335) 
+Loading checkpoint shards:  60%|██████    | 3/5 [00:16<00:11,  5.53s/it] [repeated 15x across cluster]
+(ReferenceModelRayActor pid=432926) [2025-05-27 23:59:42,530] [INFO] [config.py:734:__init__] Config mesh_device None world_size = 8
+(ActorModelRayActor pid=430510) Installed CUDA version 12.1 does not match the version torch was compiled with 12.4 but since the APIs are compatible, accepting this combination [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) Time to load cpu_adam op: 1.2110650539398193 seconds [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) Adam Optimizer #0 is created with AVX512 arithmetic capability. [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) Config: alpha=0.000001, betas=(0.900000, 0.950000), weight_decay=0.000000, adam_w=1 [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) ninja: no work to do. [repeated 3x across cluster]
+(ReferenceModelRayActor pid=432924) 
+Loading checkpoint shards: 100%|██████████| 5/5 [00:21<00:00,  3.41s/it]
+Loading checkpoint shards: 100%|██████████| 5/5 [00:21<00:00,  4.36s/it]
+(ReferenceModelRayActor pid=432929) [2025-05-27 23:59:42,537] [INFO] [config.py:734:__init__] Config mesh_device None world_size = 8
+(ReferenceModelRayActor pid=431335) Actor(
+(ReferenceModelRayActor pid=431335)   (model): Qwen2_5_VLForConditionalGeneration(
+(ReferenceModelRayActor pid=431335)     (visual): Qwen2_5_VisionTransformerPretrainedModel(
+(ReferenceModelRayActor pid=431335)       (patch_embed): Qwen2_5_VisionPatchEmbed(
+(ReferenceModelRayActor pid=431335)         (proj): Conv3d(3, 1280, kernel_size=(2, 14, 14), stride=(2, 14, 14), bias=False)
+(ReferenceModelRayActor pid=431335)       )
+(ReferenceModelRayActor pid=431335)       (rotary_pos_emb): Qwen2_5_VisionRotaryEmbedding()
+(ReferenceModelRayActor pid=431335)       (blocks): ModuleList(
+(ReferenceModelRayActor pid=431335)         (0-31): 32 x Qwen2_5_VLVisionBlock(
+(ReferenceModelRayActor pid=431335)           (norm1): Qwen2RMSNorm((0,), eps=1e-06)
+(ReferenceModelRayActor pid=431335)           (norm2): Qwen2RMSNorm((0,), eps=1e-06)
+(ReferenceModelRayActor pid=431335)           (attn): Qwen2_5_VLVisionFlashAttention2(
+(ReferenceModelRayActor pid=431335)             (qkv): Linear(in_features=1280, out_features=3840, bias=True)
+(ReferenceModelRayActor pid=431335)             (proj): Linear(in_features=1280, out_features=1280, bias=True)
+(ReferenceModelRayActor pid=431335)           )
+(ReferenceModelRayActor pid=431335)           (mlp): Qwen2_5_VLMLP(
+(ReferenceModelRayActor pid=431335)             (gate_proj): Linear(in_features=1280, out_features=3420, bias=True)
+(ReferenceModelRayActor pid=431335)             (up_proj): Linear(in_features=1280, out_features=3420, bias=True)
+(ReferenceModelRayActor pid=431335)             (down_proj): Linear(in_features=3420, out_features=1280, bias=True)
+(ReferenceModelRayActor pid=431335)             (act_fn): SiLU()
+(ReferenceModelRayActor pid=431335)           )
+(ReferenceModelRayActor pid=431335)         )
+(ReferenceModelRayActor pid=431335)       )
+(ReferenceModelRayActor pid=431335)       (merger): Qwen2_5_VLPatchMerger(
+(ReferenceModelRayActor pid=431335)         (ln_q): Qwen2RMSNorm((0,), eps=1e-06)
+(ReferenceModelRayActor pid=431335)         (mlp): Sequential(
+(ReferenceModelRayActor pid=431335)           (0): Linear(in_features=5120, out_features=5120, bias=True)
+(ReferenceModelRayActor pid=431335)           (1): GELU(approximate='none')
+(ReferenceModelRayActor pid=431335)           (2): Linear(in_features=5120, out_features=3584, bias=True)
+(ReferenceModelRayActor pid=431335)         )
+(ReferenceModelRayActor pid=431335)       )
+(ReferenceModelRayActor pid=431335)     )
+(ReferenceModelRayActor pid=431335)     (model): Qwen2_5_VLModel(
+(ReferenceModelRayActor pid=431335)       (embed_tokens): Embedding(152064, 3584)
+(ReferenceModelRayActor pid=431335)       (layers): ModuleList(
+(ReferenceModelRayActor pid=431335)         (0-27): 28 x Qwen2_5_VLDecoderLayer(
+(ReferenceModelRayActor pid=431335)           (self_attn): Qwen2_5_VLFlashAttention2(
+(ReferenceModelRayActor pid=431335)             (q_proj): Linear(in_features=3584, out_features=3584, bias=True)
+(ReferenceModelRayActor pid=431335)             (k_proj): Linear(in_features=3584, out_features=512, bias=True)
+(ReferenceModelRayActor pid=431335)             (v_proj): Linear(in_features=3584, out_features=512, bias=True)
+(ReferenceModelRayActor pid=431335)             (o_proj): Linear(in_features=3584, out_features=3584, bias=False)
+(ReferenceModelRayActor pid=431335)             (rotary_emb): Qwen2_5_VLRotaryEmbedding()
+(ReferenceModelRayActor pid=431335)           )
+(ReferenceModelRayActor pid=431335)           (mlp): Qwen2MLP(
+(ReferenceModelRayActor pid=431335)             (gate_proj): Linear(in_features=3584, out_features=18944, bias=False)
+(ReferenceModelRayActor pid=431335)             (up_proj): Linear(in_features=3584, out_features=18944, bias=False)
+(ReferenceModelRayActor pid=431335)             (down_proj): Linear(in_features=18944, out_features=3584, bias=False)
+(ReferenceModelRayActor pid=431335)             (act_fn): SiLU()
+(ReferenceModelRayActor pid=431335)           )
+(ReferenceModelRayActor pid=431335)           (input_layernorm): Qwen2RMSNorm((0,), eps=1e-06)
+(ReferenceModelRayActor pid=431335)           (post_attention_layernorm): Qwen2RMSNorm((0,), eps=1e-06)
+(ReferenceModelRayActor pid=431335)         )
+(ReferenceModelRayActor pid=431335)       )
+(ReferenceModelRayActor pid=431335)       (norm): Qwen2RMSNorm((0,), eps=1e-06)
+(ReferenceModelRayActor pid=431335)       (rotary_emb): Qwen2_5_VLRotaryEmbedding()
+(ReferenceModelRayActor pid=431335)     )
+(ReferenceModelRayActor pid=431335)     (lm_head): Linear(in_features=3584, out_features=152064, bias=False)
+(ReferenceModelRayActor pid=431335)   )
+(ReferenceModelRayActor pid=431335) )
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,009] [INFO] [logging.py:128:log_dist] [Rank 0] DeepSpeed info: version=0.16.4, git-hash=unknown, git-branch=unknown
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,009] [INFO] [comm.py:683:init_distributed] Distributed backend already initialized
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,026] [INFO] [logging.py:128:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,029] [INFO] [logging.py:128:log_dist] [Rank 0] Creating ZeRO Offload
+(ReferenceModelRayActor pid=431335) 
+Loading checkpoint shards:  80%|████████  | 4/5 [00:21<00:05,  5.30s/it] [repeated 8x across cluster]
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,231] [INFO] [utils.py:781:see_memory_usage] DeepSpeedZeRoOffload initialize [begin]
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,232] [INFO] [utils.py:782:see_memory_usage] MA 1.94 GB         Max_MA 3.98 GB         CA 4.04 GB         Max_CA 4 GB 
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,233] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 445.8 GB, percent = 44.3%
+(ReferenceModelRayActor pid=431335) Parameter Offload: Total persistent parameters: 848896 in 368 params
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,429] [INFO] [utils.py:781:see_memory_usage] DeepSpeedZeRoOffload initialize [end]
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,430] [INFO] [utils.py:782:see_memory_usage] MA 1.94 GB         Max_MA 1.94 GB         CA 4.04 GB         Max_CA 4 GB 
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,430] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 445.8 GB, percent = 44.3%
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,432] [INFO] [config.py:1001:print] DeepSpeedEngine configuration:
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,432] [INFO] [config.py:1005:print]   activation_checkpointing_config  {
+(ReferenceModelRayActor pid=431335)     "partition_activations": false, 
+(ReferenceModelRayActor pid=431335)     "contiguous_memory_optimization": false, 
+(ReferenceModelRayActor pid=431335)     "cpu_checkpointing": false, 
+(ReferenceModelRayActor pid=431335)     "number_checkpoints": null, 
+(ReferenceModelRayActor pid=431335)     "synchronize_checkpoint_boundary": false, 
+(ReferenceModelRayActor pid=431335)     "profile": false
+(ReferenceModelRayActor pid=431335) }
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,432] [INFO] [config.py:1005:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'intra_op_parallelism': 1, 'single_submit': False, 'overlap_events': True, 'use_gds': False}
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   amp_enabled .................. False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   amp_params ................... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   autotuning_config ............ {
+(ReferenceModelRayActor pid=431335)     "enabled": false, 
+(ReferenceModelRayActor pid=431335)     "start_step": null, 
+(ReferenceModelRayActor pid=431335)     "end_step": null, 
+(ReferenceModelRayActor pid=431335)     "metric_path": null, 
+(ReferenceModelRayActor pid=431335)     "arg_mappings": null, 
+(ReferenceModelRayActor pid=431335)     "metric": "throughput", 
+(ReferenceModelRayActor pid=431335)     "model_info": null, 
+(ReferenceModelRayActor pid=431335)     "results_dir": "autotuning_results", 
+(ReferenceModelRayActor pid=431335)     "exps_dir": "autotuning_exps", 
+(ReferenceModelRayActor pid=431335)     "overwrite": true, 
+(ReferenceModelRayActor pid=431335)     "fast": true, 
+(ReferenceModelRayActor pid=431335)     "start_profile_step": 3, 
+(ReferenceModelRayActor pid=431335)     "end_profile_step": 5, 
+(ReferenceModelRayActor pid=431335)     "tuner_type": "gridsearch", 
+(ReferenceModelRayActor pid=431335)     "tuner_early_stopping": 5, 
+(ReferenceModelRayActor pid=431335)     "tuner_num_trials": 50, 
+(ReferenceModelRayActor pid=431335)     "model_info_path": null, 
+(ReferenceModelRayActor pid=431335)     "mp_size": 1, 
+(ReferenceModelRayActor pid=431335)     "max_train_batch_size": null, 
+(ReferenceModelRayActor pid=431335)     "min_train_batch_size": 1, 
+(ReferenceModelRayActor pid=431335)     "max_train_micro_batch_size_per_gpu": 1.024000e+03, 
+(ReferenceModelRayActor pid=431335)     "min_train_micro_batch_size_per_gpu": 1, 
+(ReferenceModelRayActor pid=431335)     "num_tuning_micro_batch_sizes": 3
+(ReferenceModelRayActor pid=431335) }
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   bfloat16_enabled ............. True
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   bfloat16_immediate_grad_update  False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   checkpoint_parallel_write_pipeline  False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   checkpoint_tag_validation_enabled  True
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   checkpoint_tag_validation_fail  False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x7f89c8391cc0>
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   communication_data_type ...... None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   curriculum_enabled_legacy .... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   curriculum_params_legacy ..... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}}
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   data_efficiency_enabled ...... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   dataloader_drop_last ......... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   disable_allgather ............ False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   dump_state ................... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   dynamic_loss_scale_args ...... None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,433] [INFO] [config.py:1005:print]   eigenvalue_enabled ........... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   eigenvalue_gas_boundary_resolution  1
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   eigenvalue_layer_name ........ bert.encoder.layer
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   eigenvalue_layer_num ......... 0
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   eigenvalue_max_iter .......... 100
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   eigenvalue_stability ......... 1e-06
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   eigenvalue_tol ............... 0.01
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   eigenvalue_verbose ........... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   elasticity_enabled ........... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   flops_profiler_config ........ {
+(ReferenceModelRayActor pid=431335)     "enabled": false, 
+(ReferenceModelRayActor pid=431335)     "recompute_fwd_factor": 0.0, 
+(ReferenceModelRayActor pid=431335)     "profile_step": 1, 
+(ReferenceModelRayActor pid=431335)     "module_depth": -1, 
+(ReferenceModelRayActor pid=431335)     "top_modules": 1, 
+(ReferenceModelRayActor pid=431335)     "detailed": true, 
+(ReferenceModelRayActor pid=431335)     "output_file": null
+(ReferenceModelRayActor pid=431335) }
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   fp16_auto_cast ............... None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   fp16_enabled ................. False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   fp16_master_weights_and_gradients  False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   global_rank .................. 0
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   grad_accum_dtype ............. None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   gradient_accumulation_steps .. 8
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   gradient_clipping ............ 1.0
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   gradient_predivide_factor .... 1.0
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   graph_harvesting ............. False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   hybrid_engine ................ enabled=False max_out_tokens=512 inference_tp_size=1 release_inference_cache=False pin_parameters=True tp_gather_partition_size=8
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   initial_dynamic_scale ........ 1
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,434] [INFO] [config.py:1005:print]   load_universal_checkpoint .... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   loss_scale ................... 1.0
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   memory_breakdown ............. False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   mics_hierarchial_params_gather  False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   mics_shard_size .............. -1
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   monitor_config ............... tensorboard=TensorBoardConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') comet=CometConfig(enabled=False, samples_log_interval=100, project=None, workspace=None, api_key=None, experiment_name=None, experiment_key=None, online=None, mode=None) wandb=WandbConfig(enabled=False, group=None, team=None, project='deepspeed') csv_monitor=CSVConfig(enabled=False, output_path='', job_name='DeepSpeedJobName')
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   nebula_config ................ {
+(ReferenceModelRayActor pid=431335)     "enabled": false, 
+(ReferenceModelRayActor pid=431335)     "persistent_storage_path": null, 
+(ReferenceModelRayActor pid=431335)     "persistent_time_interval": 100, 
+(ReferenceModelRayActor pid=431335)     "num_of_version_in_retention": 2, 
+(ReferenceModelRayActor pid=431335)     "enable_nebula_load": true, 
+(ReferenceModelRayActor pid=431335)     "load_path": null
+(ReferenceModelRayActor pid=431335) }
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   optimizer_legacy_fusion ...... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   optimizer_name ............... None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   optimizer_params ............. None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0, 'pipe_partitioned': True, 'grad_partitioned': True}
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   pld_enabled .................. False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   pld_params ................... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   prescale_gradients ........... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   scheduler_name ............... None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   scheduler_params ............. None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   seq_parallel_communication_data_type  torch.float32
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   sparse_attention ............. None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   sparse_gradients_enabled ..... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   steps_per_print .............. 100
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   tensor_parallel_config ....... dtype=torch.float16 autotp_size=0 tensor_parallel=TPConfig(tp_size=1, tp_grain_size=1, mpu=None, tp_group=None) injection_policy_tuple=None keep_module_on_host=False replace_with_kernel_inject=False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   timers_config ................ enabled=True synchronized=True
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,435] [INFO] [config.py:1005:print]   train_batch_size ............. 128
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   train_micro_batch_size_per_gpu  2
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   use_data_before_expert_parallel_  False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   use_node_local_storage ....... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   wall_clock_breakdown ......... False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   weight_quantization_config ... None
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   world_size ................... 8
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   zero_allow_untested_optimizer  False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   zero_config .................. stage=3 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 use_multi_rank_bucket_allreduce=True allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=DeepSpeedZeroOffloadParamConfig(device='none', nvme_path=None, buffer_count=5, buffer_size=100000000, max_in_cpu=1000000000, pin_memory=True) offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False module_granularity_threshold=0 use_all_reduce_for_fetch_params=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False zero_hpz_partition_size=1 zero_quantized_weights=False zero_quantized_nontrainable_weights=False zero_quantized_gradients=False zeropp_loco_param=None mics_shard_size=-1 mics_hierarchical_params_gather=False memory_efficient_linear=True pipeline_loading_checkpoint=False override_module_apply=True log_trace_cache_warnings=False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   zero_enabled ................. True
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   zero_force_ds_cpu_optimizer .. True
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:1005:print]   zero_optimization_stage ...... 3
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,436] [INFO] [config.py:991:print_user_config]   json = {
+(ReferenceModelRayActor pid=431335)     "steps_per_print": 100, 
+(ReferenceModelRayActor pid=431335)     "zero_optimization": {
+(ReferenceModelRayActor pid=431335)         "stage": 3, 
+(ReferenceModelRayActor pid=431335)         "stage3_max_live_parameters": "auto", 
+(ReferenceModelRayActor pid=431335)         "stage3_max_reuse_distance": "auto", 
+(ReferenceModelRayActor pid=431335)         "stage3_param_persistence_threshold": "auto", 
+(ReferenceModelRayActor pid=431335)         "stage3_prefetch_bucket_size": "auto", 
+(ReferenceModelRayActor pid=431335)         "offload_param": {
+(ReferenceModelRayActor pid=431335)             "device": "none", 
+(ReferenceModelRayActor pid=431335)             "pin_memory": true
+(ReferenceModelRayActor pid=431335)         }
+(ReferenceModelRayActor pid=431335)     }, 
+(ReferenceModelRayActor pid=431335)     "bf16": {
+(ReferenceModelRayActor pid=431335)         "enabled": true
+(ReferenceModelRayActor pid=431335)     }, 
+(ReferenceModelRayActor pid=431335)     "gradient_clipping": 1.0, 
+(ReferenceModelRayActor pid=431335)     "prescale_gradients": false, 
+(ReferenceModelRayActor pid=431335)     "wall_clock_breakdown": false, 
+(ReferenceModelRayActor pid=431335)     "train_micro_batch_size_per_gpu": 2, 
+(ReferenceModelRayActor pid=431335)     "train_batch_size": 128
+(ReferenceModelRayActor pid=431335) }
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ActorModelRayActor pid=431329) in preprocess_data None False
+(ReferenceModelRayActor pid=431335) [2025-05-27 23:59:44,009] [INFO] [config.py:734:__init__] Config mesh_device None world_size = 8 [repeated 6x across cluster]
+(ActorModelRayActor pid=430510) loaded /mnt/petrelfs/luyiting/MultiAgentEval/data_process_v1/train_ava_mini_evalmuse_koniq_llavastyle_openrlhf_merged.jsonl with data_files=/mnt/petrelfs/luyiting/MultiAgentEval/data_process_v1/train_ava_mini_evalmuse_koniq_llavastyle_openrlhf_merged.jsonl
+(ActorModelRayActor pid=430510) [Dataset({
+(ActorModelRayActor pid=430510)     features: ['message', 'answer'],
+(ActorModelRayActor pid=430510)     num_rows: 24000
+(ActorModelRayActor pid=430510) })]
+(ActorModelRayActor pid=430510) 
+Preprocessing data:   0%|          | 0/24000 [00:00<?, ?it/s]
+(ReferenceModelRayActor pid=431335) 
+Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00,  3.80s/it]
+Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00,  4.59s/it] [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) 
+Preprocessing data:  12%|█▏        | 2766/24000 [00:00<00:00, 27652.44it/s]
+Preprocessing data:  23%|██▎       | 5631/24000 [00:00<00:00, 28232.38it/s]
+(ActorModelRayActor pid=430510) 
+Preprocessing data:  35%|███▌      | 8503/24000 [00:00<00:00, 28452.78it/s]
+Preprocessing data:  47%|████▋     | 11349/24000 [00:00<00:00, 28184.35it/s]
+(ActorModelRayActor pid=430510) 
+Preprocessing data:  59%|█████▉    | 14168/24000 [00:00<00:00, 27719.16it/s]
+Preprocessing data:  71%|███████   | 16970/24000 [00:00<00:00, 27816.93it/s]
+(ActorModelRayActor pid=430510) 
+Preprocessing data:  82%|████████▏ | 19753/24000 [00:00<00:00, 27600.31it/s]
+Preprocessing data:  94%|█████████▍| 22521/24000 [00:00<00:00, 27624.13it/s]
+Preprocessing data: 100%|██████████| 24000/24000 [00:00<00:00, 27816.32it/s]
+(ActorModelRayActor pid=430510) in preprocess_data None False [repeated 188000x across cluster]
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,442] [INFO] [logging.py:128:log_dist] [Rank 0] DeepSpeed info: version=0.16.4, git-hash=unknown, git-branch=unknown
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,443] [INFO] [comm.py:683:init_distributed] Distributed backend already initialized
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,458] [INFO] [logging.py:128:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,460] [INFO] [logging.py:128:log_dist] [Rank 0] Using client Optimizer as basic optimizer
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,460] [INFO] [logging.py:128:log_dist] [Rank 0] Removing param_group that has no 'params' in the basic Optimizer
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,501] [INFO] [logging.py:128:log_dist] [Rank 0] DeepSpeed Basic Optimizer = DeepSpeedCPUAdam
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,501] [INFO] [utils.py:59:is_zero_supported_optimizer] Checking ZeRO support for optimizer=DeepSpeedCPUAdam type=<class 'deepspeed.ops.adam.cpu_adam.DeepSpeedCPUAdam'>
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,501] [INFO] [logging.py:128:log_dist] [Rank 0] Creating fp16 ZeRO stage 3 optimizer, MiCS is enabled False, Hierarchical params gather False
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,501] [INFO] [logging.py:128:log_dist] [Rank 0] Creating torch.bfloat16 ZeRO stage 3 optimizer
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,722] [INFO] [utils.py:781:see_memory_usage] Stage 3 initialize beginning
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,723] [INFO] [utils.py:782:see_memory_usage] MA 1.94 GB         Max_MA 3.98 GB         CA 4.04 GB         Max_CA 4 GB 
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,723] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 445.88 GB, percent = 44.3%
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,726] [INFO] [stage3.py:170:__init__] Reduce bucket size 500000000
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,726] [INFO] [stage3.py:171:__init__] Prefetch bucket size 50000000
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,906] [INFO] [utils.py:781:see_memory_usage] DeepSpeedZeRoOffload initialize [begin]
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,907] [INFO] [utils.py:782:see_memory_usage] MA 1.94 GB         Max_MA 1.94 GB         CA 4.04 GB         Max_CA 4 GB 
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,907] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 445.88 GB, percent = 44.3%
+(ActorModelRayActor pid=430510) Parameter Offload: Total persistent parameters: 848896 in 368 params
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:49,112] [INFO] [utils.py:781:see_memory_usage] DeepSpeedZeRoOffload initialize [end]
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:49,113] [INFO] [utils.py:782:see_memory_usage] MA 1.94 GB         Max_MA 1.94 GB         CA 4.04 GB         Max_CA 4 GB 
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:49,113] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 445.88 GB, percent = 44.3%
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:49,289] [INFO] [utils.py:781:see_memory_usage] Before creating fp16 partitions
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:49,290] [INFO] [utils.py:782:see_memory_usage] MA 1.94 GB         Max_MA 1.94 GB         CA 4.04 GB         Max_CA 4 GB 
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:49,291] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 445.88 GB, percent = 44.3%
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:51,659] [INFO] [utils.py:781:see_memory_usage] After creating fp16 partitions: 2
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:51,660] [INFO] [utils.py:782:see_memory_usage] MA 1.93 GB         Max_MA 1.94 GB         CA 1.94 GB         Max_CA 4 GB 
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:51,661] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 449.43 GB, percent = 44.6%
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:51,872] [INFO] [utils.py:781:see_memory_usage] Before creating fp32 partitions
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:51,873] [INFO] [utils.py:782:see_memory_usage] MA 1.93 GB         Max_MA 1.93 GB         CA 1.94 GB         Max_CA 2 GB 
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:51,873] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 451.83 GB, percent = 44.9%
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:55,698] [INFO] [utils.py:781:see_memory_usage] After creating fp32 partitions
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:55,698] [INFO] [utils.py:782:see_memory_usage] MA 1.93 GB         Max_MA 1.93 GB         CA 1.94 GB         Max_CA 2 GB 
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:55,699] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 478.68 GB, percent = 47.5%
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:48,443] [INFO] [config.py:734:__init__] Config mesh_device None world_size = 8 [repeated 8x across cluster]
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:56,077] [INFO] [utils.py:781:see_memory_usage] Before initializing optimizer states
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:56,078] [INFO] [utils.py:782:see_memory_usage] MA 1.93 GB         Max_MA 1.93 GB         CA 1.94 GB         Max_CA 2 GB 
+(ActorModelRayActor pid=430510) [2025-05-27 23:59:56,078] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 481.26 GB, percent = 47.8%
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:04,652] [INFO] [utils.py:781:see_memory_usage] After initializing optimizer states
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:04,652] [INFO] [utils.py:782:see_memory_usage] MA 1.93 GB         Max_MA 1.93 GB         CA 1.94 GB         Max_CA 2 GB 
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:04,654] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 509.36 GB, percent = 50.6%
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:04,655] [INFO] [stage3.py:534:_setup_for_real_optimizer] optimizer state initialized
+(ActorModelRayActor pid=430510) in preprocess_data None False [repeated 3000x across cluster]
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,264] [INFO] [utils.py:781:see_memory_usage] After initializing ZeRO optimizer
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,265] [INFO] [utils.py:782:see_memory_usage] MA 2.86 GB         Max_MA 4.89 GB         CA 5.02 GB         Max_CA 5 GB 
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,266] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 525.83 GB, percent = 52.2%
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,266] [INFO] [logging.py:128:log_dist] [Rank 0] DeepSpeed Final Optimizer = DeepSpeedZeroOptimizer_Stage3
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,266] [INFO] [logging.py:128:log_dist] [Rank 0] DeepSpeed using client LR scheduler
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,266] [INFO] [logging.py:128:log_dist] [Rank 0] DeepSpeed LR Scheduler = <torch.optim.lr_scheduler.LambdaLR object at 0x7f18fcf87610>
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,266] [INFO] [logging.py:128:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0, 0.0], mom=[(0.9, 0.95), (0.9, 0.95)]
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,268] [INFO] [config.py:1001:print] DeepSpeedEngine configuration:
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   activation_checkpointing_config  {
+(ActorModelRayActor pid=430510)     "partition_activations": false, 
+(ActorModelRayActor pid=430510)     "contiguous_memory_optimization": false, 
+(ActorModelRayActor pid=430510)     "cpu_checkpointing": false, 
+(ActorModelRayActor pid=430510)     "number_checkpoints": null, 
+(ActorModelRayActor pid=430510)     "synchronize_checkpoint_boundary": false, 
+(ActorModelRayActor pid=430510)     "profile": false
+(ActorModelRayActor pid=430510) }
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'intra_op_parallelism': 1, 'single_submit': False, 'overlap_events': True, 'use_gds': False}
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   amp_enabled .................. False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   amp_params ................... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   autotuning_config ............ {
+(ActorModelRayActor pid=430510)     "enabled": false, 
+(ActorModelRayActor pid=430510)     "start_step": null, 
+(ActorModelRayActor pid=430510)     "end_step": null, 
+(ActorModelRayActor pid=430510)     "metric_path": null, 
+(ActorModelRayActor pid=430510)     "arg_mappings": null, 
+(ActorModelRayActor pid=430510)     "metric": "throughput", 
+(ActorModelRayActor pid=430510)     "model_info": null, 
+(ActorModelRayActor pid=430510)     "results_dir": "autotuning_results", 
+(ActorModelRayActor pid=430510)     "exps_dir": "autotuning_exps", 
+(ActorModelRayActor pid=430510)     "overwrite": true, 
+(ActorModelRayActor pid=430510)     "fast": true, 
+(ActorModelRayActor pid=430510)     "start_profile_step": 3, 
+(ActorModelRayActor pid=430510)     "end_profile_step": 5, 
+(ActorModelRayActor pid=430510)     "tuner_type": "gridsearch", 
+(ActorModelRayActor pid=430510)     "tuner_early_stopping": 5, 
+(ActorModelRayActor pid=430510)     "tuner_num_trials": 50, 
+(ActorModelRayActor pid=430510)     "model_info_path": null, 
+(ActorModelRayActor pid=430510)     "mp_size": 1, 
+(ActorModelRayActor pid=430510)     "max_train_batch_size": null, 
+(ActorModelRayActor pid=430510)     "min_train_batch_size": 1, 
+(ActorModelRayActor pid=430510)     "max_train_micro_batch_size_per_gpu": 1.024000e+03, 
+(ActorModelRayActor pid=430510)     "min_train_micro_batch_size_per_gpu": 1, 
+(ActorModelRayActor pid=430510)     "num_tuning_micro_batch_sizes": 3
+(ActorModelRayActor pid=430510) }
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   bfloat16_enabled ............. True
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   bfloat16_immediate_grad_update  False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   checkpoint_parallel_write_pipeline  False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   checkpoint_tag_validation_enabled  True
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   checkpoint_tag_validation_fail  False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x7f18fcf87580>
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,269] [INFO] [config.py:1005:print]   communication_data_type ...... None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   curriculum_enabled_legacy .... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   curriculum_params_legacy ..... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}}
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   data_efficiency_enabled ...... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   dataloader_drop_last ......... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   disable_allgather ............ False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   dump_state ................... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   dynamic_loss_scale_args ...... None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   eigenvalue_enabled ........... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   eigenvalue_gas_boundary_resolution  1
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   eigenvalue_layer_name ........ bert.encoder.layer
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   eigenvalue_layer_num ......... 0
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   eigenvalue_max_iter .......... 100
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   eigenvalue_stability ......... 1e-06
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   eigenvalue_tol ............... 0.01
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   eigenvalue_verbose ........... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   elasticity_enabled ........... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,270] [INFO] [config.py:1005:print]   flops_profiler_config ........ {
+(ActorModelRayActor pid=430510)     "enabled": false, 
+(ActorModelRayActor pid=430510)     "recompute_fwd_factor": 0.0, 
+(ActorModelRayActor pid=430510)     "profile_step": 1, 
+(ActorModelRayActor pid=430510)     "module_depth": -1, 
+(ActorModelRayActor pid=430510)     "top_modules": 1, 
+(ActorModelRayActor pid=430510)     "detailed": true, 
+(ActorModelRayActor pid=430510)     "output_file": null
+(ActorModelRayActor pid=430510) }
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   fp16_auto_cast ............... None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   fp16_enabled ................. False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   fp16_master_weights_and_gradients  False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   global_rank .................. 0
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   grad_accum_dtype ............. None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   gradient_accumulation_steps .. 8
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   gradient_clipping ............ 1.0
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   gradient_predivide_factor .... 1.0
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   graph_harvesting ............. False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   hybrid_engine ................ enabled=False max_out_tokens=512 inference_tp_size=1 release_inference_cache=False pin_parameters=True tp_gather_partition_size=8
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   initial_dynamic_scale ........ 1
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   load_universal_checkpoint .... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   loss_scale ................... 1.0
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   memory_breakdown ............. False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   mics_hierarchial_params_gather  False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   mics_shard_size .............. -1
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   monitor_config ............... tensorboard=TensorBoardConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') comet=CometConfig(enabled=False, samples_log_interval=100, project=None, workspace=None, api_key=None, experiment_name=None, experiment_key=None, online=None, mode=None) wandb=WandbConfig(enabled=False, group=None, team=None, project='deepspeed') csv_monitor=CSVConfig(enabled=False, output_path='', job_name='DeepSpeedJobName')
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   nebula_config ................ {
+(ActorModelRayActor pid=430510)     "enabled": false, 
+(ActorModelRayActor pid=430510)     "persistent_storage_path": null, 
+(ActorModelRayActor pid=430510)     "persistent_time_interval": 100, 
+(ActorModelRayActor pid=430510)     "num_of_version_in_retention": 2, 
+(ActorModelRayActor pid=430510)     "enable_nebula_load": true, 
+(ActorModelRayActor pid=430510)     "load_path": null
+(ActorModelRayActor pid=430510) }
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   optimizer_legacy_fusion ...... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,271] [INFO] [config.py:1005:print]   optimizer_name ............... None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   optimizer_params ............. None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0, 'pipe_partitioned': True, 'grad_partitioned': True}
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   pld_enabled .................. False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   pld_params ................... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   prescale_gradients ........... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   scheduler_name ............... None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   scheduler_params ............. None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   seq_parallel_communication_data_type  torch.float32
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   sparse_attention ............. None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   sparse_gradients_enabled ..... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   steps_per_print .............. 100
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   tensor_parallel_config ....... dtype=torch.float16 autotp_size=0 tensor_parallel=TPConfig(tp_size=1, tp_grain_size=1, mpu=None, tp_group=None) injection_policy_tuple=None keep_module_on_host=False replace_with_kernel_inject=False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   timers_config ................ enabled=True synchronized=True
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   train_batch_size ............. 128
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   train_micro_batch_size_per_gpu  2
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   use_data_before_expert_parallel_  False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   use_node_local_storage ....... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   wall_clock_breakdown ......... False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   weight_quantization_config ... None
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   world_size ................... 8
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,272] [INFO] [config.py:1005:print]   zero_allow_untested_optimizer  False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,273] [INFO] [config.py:1005:print]   zero_config .................. stage=3 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 use_multi_rank_bucket_allreduce=True allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=DeepSpeedZeroOffloadParamConfig(device='none', nvme_path=None, buffer_count=5, buffer_size=100000000, max_in_cpu=1000000000, pin_memory=False) offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='cpu', nvme_path=None, buffer_count=4, pin_memory=True, pipeline_read=False, pipeline_write=False, fast_init=False, ratio=1.0) sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False module_granularity_threshold=0 use_all_reduce_for_fetch_params=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False zero_hpz_partition_size=1 zero_quantized_weights=False zero_quantized_nontrainable_weights=False zero_quantized_gradients=False zeropp_loco_param=None mics_shard_size=-1 mics_hierarchical_params_gather=False memory_efficient_linear=True pipeline_loading_checkpoint=False override_module_apply=True log_trace_cache_warnings=False
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,273] [INFO] [config.py:1005:print]   zero_enabled ................. True
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,273] [INFO] [config.py:1005:print]   zero_force_ds_cpu_optimizer .. True
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,273] [INFO] [config.py:1005:print]   zero_optimization_stage ...... 3
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,273] [INFO] [config.py:991:print_user_config]   json = {
+(ActorModelRayActor pid=430510)     "steps_per_print": 100, 
+(ActorModelRayActor pid=430510)     "zero_optimization": {
+(ActorModelRayActor pid=430510)         "stage": 3, 
+(ActorModelRayActor pid=430510)         "offload_param": {
+(ActorModelRayActor pid=430510)             "device": "none"
+(ActorModelRayActor pid=430510)         }, 
+(ActorModelRayActor pid=430510)         "offload_optimizer": {
+(ActorModelRayActor pid=430510)             "device": "cpu", 
+(ActorModelRayActor pid=430510)             "pin_memory": true
+(ActorModelRayActor pid=430510)         }, 
+(ActorModelRayActor pid=430510)         "sub_group_size": "auto", 
+(ActorModelRayActor pid=430510)         "stage3_max_live_parameters": "auto", 
+(ActorModelRayActor pid=430510)         "stage3_max_reuse_distance": "auto", 
+(ActorModelRayActor pid=430510)         "stage3_param_persistence_threshold": "auto", 
+(ActorModelRayActor pid=430510)         "stage3_prefetch_bucket_size": "auto", 
+(ActorModelRayActor pid=430510)         "reduce_bucket_size": "auto", 
+(ActorModelRayActor pid=430510)         "zero_hpz_partition_size": 1, 
+(ActorModelRayActor pid=430510)         "zero_quantized_weights": false, 
+(ActorModelRayActor pid=430510)         "zero_quantized_gradients": false, 
+(ActorModelRayActor pid=430510)         "reduce_scatter": true
+(ActorModelRayActor pid=430510)     }, 
+(ActorModelRayActor pid=430510)     "bf16": {
+(ActorModelRayActor pid=430510)         "enabled": true
+(ActorModelRayActor pid=430510)     }, 
+(ActorModelRayActor pid=430510)     "gradient_clipping": 1.0, 
+(ActorModelRayActor pid=430510)     "prescale_gradients": false, 
+(ActorModelRayActor pid=430510)     "wall_clock_breakdown": false, 
+(ActorModelRayActor pid=430510)     "data_types": {
+(ActorModelRayActor pid=430510)         "grad_accum_dtype": null
+(ActorModelRayActor pid=430510)     }, 
+(ActorModelRayActor pid=430510)     "checkpoint": {
+(ActorModelRayActor pid=430510)         "load_universal": false
+(ActorModelRayActor pid=430510)     }, 
+(ActorModelRayActor pid=430510)     "train_micro_batch_size_per_gpu": 2, 
+(ActorModelRayActor pid=430510)     "train_batch_size": 128
+(ActorModelRayActor pid=430510) }
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,284] [INFO] [torch_checkpoint_engine.py:27:load] [Torch] Loading checkpoint from /mnt/petrelfs/luyiting/MultiAgentEval/checkpoints_lmmr1//lmm-r1-ava-evalmuse-koniq-grpo-score-7B/ckpt/_actor/global_step60/zero_pp_rank_0_mp_rank_00_model_states.pt...
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,313] [INFO] [torch_checkpoint_engine.py:29:load] [Torch] Loaded checkpoint from /mnt/petrelfs/luyiting/MultiAgentEval/checkpoints_lmmr1//lmm-r1-ava-evalmuse-koniq-grpo-score-7B/ckpt/_actor/global_step60/zero_pp_rank_0_mp_rank_00_model_states.pt.
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,314] [INFO] [torch_checkpoint_engine.py:27:load] [Torch] Loading checkpoint from /mnt/petrelfs/luyiting/MultiAgentEval/checkpoints_lmmr1//lmm-r1-ava-evalmuse-koniq-grpo-score-7B/ckpt/_actor/global_step60/zero_pp_rank_0_mp_rank_00_model_states.pt...
+(ActorModelRayActor pid=430510) [2025-05-28 00:00:07,338] [INFO] [torch_checkpoint_engine.py:29:load] [Torch] Loaded checkpoint from /mnt/petrelfs/luyiting/MultiAgentEval/checkpoints_lmmr1//lmm-r1-ava-evalmuse-koniq-grpo-score-7B/ckpt/_actor/global_step60/zero_pp_rank_0_mp_rank_00_model_states.pt.
+(ActorModelRayActor pid=431333) [2025-05-28 00:00:33,844] [INFO] [engine.py:3185:_get_all_zero_checkpoint_state_dicts] successfully read 8 ZeRO state_dicts for rank 1
+(ActorModelRayActor pid=431330) [2025-05-28 00:00:11,542] [INFO] [torch_checkpoint_engine.py:27:load] [Torch] Loading checkpoint from /mnt/petrelfs/luyiting/MultiAgentEval/checkpoints_lmmr1//lmm-r1-ava-evalmuse-koniq-grpo-score-7B/ckpt/_actor/global_step60/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... [repeated 22x across cluster]
+(ActorModelRayActor pid=431333) [2025-05-28 00:00:33,844] [INFO] [torch_checkpoint_engine.py:29:load] [Torch] Loaded checkpoint from /mnt/petrelfs/luyiting/MultiAgentEval/checkpoints_lmmr1//lmm-r1-ava-evalmuse-koniq-grpo-score-7B/ckpt/_actor/global_step60/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. [repeated 15x across cluster]
+(ActorModelRayActor pid=431334) [2025-05-28 00:00:36,557] [INFO] [engine.py:3135:_load_zero_checkpoint] loading 8 zero partition checkpoints for rank 4
+(ActorModelRayActor pid=431328) [2025-05-28 00:00:40,539] [INFO] [engine.py:3185:_get_all_zero_checkpoint_state_dicts] successfully read 8 ZeRO state_dicts for rank 3 [repeated 2x across cluster]
+(ActorModelRayActor pid=431328) [2025-05-28 00:00:40,538] [INFO] [torch_checkpoint_engine.py:29:load] [Torch] Loaded checkpoint from /mnt/petrelfs/luyiting/MultiAgentEval/checkpoints_lmmr1//lmm-r1-ava-evalmuse-koniq-grpo-score-7B/ckpt/_actor/global_step60/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. [repeated 2x across cluster]
+(ActorModelRayActor pid=431333) [2025-05-28 00:00:36,757] [INFO] [engine.py:3135:_load_zero_checkpoint] loading 8 zero partition checkpoints for rank 1
+(ActorModelRayActor pid=431328) [2025-05-28 00:00:43,509] [INFO] [engine.py:3135:_load_zero_checkpoint] loading 8 zero partition checkpoints for rank 3
+(ActorModelRayActor pid=431332) [2025-05-28 00:00:44,685] [INFO] [engine.py:3185:_get_all_zero_checkpoint_state_dicts] successfully read 8 ZeRO state_dicts for rank 5 [repeated 5x across cluster]
+(ActorModelRayActor pid=431332) [2025-05-28 00:00:44,685] [INFO] [torch_checkpoint_engine.py:29:load] [Torch] Loaded checkpoint from /mnt/petrelfs/luyiting/MultiAgentEval/checkpoints_lmmr1//lmm-r1-ava-evalmuse-koniq-grpo-score-7B/ckpt/_actor/global_step60/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. [repeated 5x across cluster]
+(ActorModelRayActor pid=430510) Loaded the checkpoint: /mnt/petrelfs/luyiting/MultiAgentEval/checkpoints_lmmr1//lmm-r1-ava-evalmuse-koniq-grpo-score-7B/ckpt/_actor, consumed_samples: 7680
+(ActorModelRayActor pid=430510) wandb: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.
+(ActorModelRayActor pid=430510) wandb: Tracking run with wandb version 0.19.8
+(ActorModelRayActor pid=430510) wandb: W&B syncing is set to `offline` in this directory.  
+(ActorModelRayActor pid=430510) wandb: Run `wandb online` or set WANDB_MODE=online to enable cloud syncing.
+(LLMRayActor pid=425166) init_process_group: master_address=10.140.0.151, master_port=6188,  rank=1, world_size=9, group_name=openrlhf
+(ActorModelRayActor pid=431332) [2025-05-28 00:00:47,458] [INFO] [engine.py:3135:_load_zero_checkpoint] loading 8 zero partition checkpoints for rank 5 [repeated 5x across cluster]
+(LLMRayActor pid=425159) INFO 05-28 00:00:52 [executor_base.py:219] It took 1.357901 seconds to wake up.
+(LLMRayActor pid=425166) update weight: visual.patch_embed.proj.weight, dtype: torch.bfloat16, shape: torch.Size([1280, 3, 2, 14, 14])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.norm1.weight, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.norm2.weight, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.attn.qkv.weight, dtype: torch.bfloat16, shape: torch.Size([3840, 1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.attn.qkv.bias, dtype: torch.bfloat16, shape: torch.Size([3840])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.attn.proj.weight, dtype: torch.bfloat16, shape: torch.Size([1280, 1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.attn.proj.bias, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.mlp.gate_proj.weight, dtype: torch.bfloat16, shape: torch.Size([3420, 1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.mlp.gate_proj.bias, dtype: torch.bfloat16, shape: torch.Size([3420])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.mlp.up_proj.weight, dtype: torch.bfloat16, shape: torch.Size([3420, 1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.mlp.up_proj.bias, dtype: torch.bfloat16, shape: torch.Size([3420])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.mlp.down_proj.weight, dtype: torch.bfloat16, shape: torch.Size([1280, 3420])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.mlp.down_proj.bias, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.1.norm1.weight, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.1.norm2.weight, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.1.attn.qkv.weight, dtype: torch.bfloat16, shape: torch.Size([3840, 1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.1.attn.qkv.bias, dtype: torch.bfloat16, shape: torch.Size([3840])
+(LLMRayActor pid=425166) update weight: visual.blocks.1.attn.proj.weight, dtype: torch.bfloat16, shape: torch.Size([1280, 1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.1.attn.proj.bias, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.1.mlp.gate_proj.weight, dtype: torch.bfloat16, shape: torch.Size([3420, 1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.1.mlp.gate_proj.bias, dtype: torch.bfloat16, shape: torch.Size([3420])
+(LLMRayActor pid=425166) update weight: visual.blocks.1.mlp.up_proj.weight, dtype: torch.bfloat16, shape: torch.Size([3420, 1280])
+(LLMRayActor pid=425165) init_process_group: master_address=10.140.0.151, master_port=6188,  rank=8, world_size=9, group_name=openrlhf [repeated 7x across cluster]
+(LLMRayActor pid=425166) update weight: visual.merger.ln_q.weight, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: model.embed_tokens.weight, dtype: torch.bfloat16, shape: torch.Size([152064, 3584])
+(LLMRayActor pid=425165) INFO 05-28 00:00:54 [executor_base.py:219] It took 3.127167 seconds to wake up. [repeated 7x across cluster]
+(LLMRayActor pid=425165) update weight: visual.patch_embed.proj.weight, dtype: torch.bfloat16, shape: torch.Size([1280, 3, 2, 14, 14]) [repeated 7x across cluster]
+(LLMRayActor pid=425166) update weight: model.layers.0.mlp.gate_proj.weight, dtype: torch.bfloat16, shape: torch.Size([18944, 3584]) [repeated 3140x across cluster]
+(LLMRayActor pid=425165) update weight: visual.merger.ln_q.weight, dtype: torch.bfloat16, shape: torch.Size([1280]) [repeated 7x across cluster]
+(LLMRayActor pid=425165) update weight: model.embed_tokens.weight, dtype: torch.bfloat16, shape: torch.Size([152064, 3584]) [repeated 7x across cluster]
+(LLMRayActor pid=425166) update weight: model.layers.10.mlp.gate_proj.weight, dtype: torch.bfloat16, shape: torch.Size([18944, 3584]) [repeated 897x across cluster]
+(LLMRayActor pid=425166) update weight: model.layers.21.mlp.down_proj.weight, dtype: torch.bfloat16, shape: torch.Size([3584, 18944]) [repeated 1135x across cluster]
+(LLMRayActor pid=425166) update weight: model.norm.weight, dtype: torch.bfloat16, shape: torch.Size([3584])
+(LLMRayActor pid=425166) update weight: lm_head.weight, dtype: torch.bfloat16, shape: torch.Size([152064, 3584])
+(LLMRayActor pid=425166) INFO 05-28 00:01:14 [worker.py:133] Sleep mode freed 38.53 GiB memory, 16.25 GiB memory is still in use.
+(LLMRayActor pid=425166) INFO 05-28 00:01:14 [executor_base.py:208] It took 1.393784 seconds to fall asleep.
+(LLMRayActor pid=425164) update weight: model.layers.27.post_attention_layernorm.weight, dtype: torch.bfloat16, shape: torch.Size([3584]) [repeated 599x across cluster]
+(ActorModelRayActor pid=430510) 
+Episode [1/2]:   0%|          | 0/127 [00:00<?, ?it/s]
+(LLMRayActor pid=425166) INFO 05-28 00:01:16 [executor_base.py:219] It took 1.667116 seconds to wake up.
+(LLMRayActor pid=425165) update weight: model.norm.weight, dtype: torch.bfloat16, shape: torch.Size([3584]) [repeated 7x across cluster]
+(LLMRayActor pid=425165) update weight: lm_head.weight, dtype: torch.bfloat16, shape: torch.Size([152064, 3584]) [repeated 7x across cluster]
+(LLMRayActor pid=425159) 
+Processed prompts:   0%|          | 0/256 [00:00<?, ?it/s, est. speed input: 0.00 toks/s, output: 0.00 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:   0%|          | 0/256 [00:00<?, ?it/s, est. speed input: 0.00 toks/s, output: 0.00 toks/s] [repeated 5x across cluster]
+(LLMRayActor pid=425159) 
+Processed prompts:   0%|          | 1/256 [00:21<1:31:16, 21.47s/it, est. speed input: 72.27 toks/s, output: 3.77 toks/s]
+Processed prompts:   1%|          | 2/256 [00:21<37:45,  8.92s/it, est. speed input: 143.68 toks/s, output: 7.64 toks/s] 
+(LLMRayActor pid=425166) 
+Processed prompts:   0%|          | 0/256 [00:00<?, ?it/s, est. speed input: 0.00 toks/s, output: 0.00 toks/s] [repeated 2x across cluster]
+(LLMRayActor pid=425159) 
+Processed prompts:   1%|          | 3/256 [00:22<21:20,  5.06s/it, est. speed input: 163.37 toks/s, output: 11.78 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:   3%|▎         | 7/256 [00:22<06:01,  1.45s/it, est. speed input: 392.33 toks/s, output: 29.09 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:   4%|▍         | 11/256 [00:22<02:58,  1.37it/s, est. speed input: 524.29 toks/s, output: 47.21 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:   5%|▌         | 13/256 [00:22<02:14,  1.81it/s, est. speed input: 560.98 toks/s, output: 56.27 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:   7%|▋         | 17/256 [00:22<01:18,  3.05it/s, est. speed input: 631.91 toks/s, output: 75.12 toks/s]
+Processed prompts:   8%|▊         | 20/256 [00:22<00:57,  4.12it/s, est. speed input: 683.81 toks/s, output: 89.31 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:   9%|▉         | 23/256 [00:23<00:42,  5.54it/s, est. speed input: 740.16 toks/s, output: 104.00 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  11%|█         | 28/256 [00:23<00:26,  8.73it/s, est. speed input: 839.31 toks/s, output: 129.07 toks/s]
+Processed prompts:  13%|█▎        | 33/256 [00:23<00:18, 12.35it/s, est. speed input: 1023.59 toks/s, output: 154.46 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  16%|█▌        | 40/256 [00:23<00:11, 18.60it/s, est. speed input: 1240.13 toks/s, output: 190.83 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  17%|█▋        | 44/256 [00:23<00:10, 20.87it/s, est. speed input: 1304.73 toks/s, output: 211.56 toks/s]
+Processed prompts:  21%|██        | 53/256 [00:23<00:06, 31.00it/s, est. speed input: 1550.31 toks/s, output: 260.20 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  25%|██▌       | 64/256 [00:23<00:04, 43.41it/s, est. speed input: 1744.28 toks/s, output: 320.49 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  30%|██▉       | 76/256 [00:23<00:03, 53.40it/s, est. speed input: 1981.89 toks/s, output: 386.81 toks/s]
+Processed prompts:  33%|███▎      | 84/256 [00:24<00:03, 56.89it/s, est. speed input: 2155.51 toks/s, output: 431.65 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  38%|███▊      | 97/256 [00:24<00:02, 72.12it/s, est. speed input: 2389.47 toks/s, output: 506.56 toks/s]
+Processed prompts:  42%|████▏     | 108/256 [00:24<00:01, 79.87it/s, est. speed input: 2654.60 toks/s, output: 570.63 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  46%|████▌     | 117/256 [00:24<00:01, 76.28it/s, est. speed input: 2865.78 toks/s, output: 622.61 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  53%|█████▎    | 136/256 [00:24<00:01, 100.96it/s, est. speed input: 3253.75 toks/s, output: 738.48 toks/s]
+Processed prompts:  57%|█████▋    | 147/256 [00:24<00:01, 101.19it/s, est. speed input: 3462.19 toks/s, output: 805.69 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  62%|██████▏   | 158/256 [00:24<00:00, 103.10it/s, est. speed input: 3693.57 toks/s, output: 874.41 toks/s]
+Processed prompts:  68%|██████▊   | 173/256 [00:24<00:00, 115.88it/s, est. speed input: 3977.63 toks/s, output: 971.14 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  73%|███████▎  | 187/256 [00:24<00:00, 117.77it/s, est. speed input: 4247.80 toks/s, output: 1062.29 toks/s]
+Processed prompts:  81%|████████  | 207/256 [00:25<00:00, 139.11it/s, est. speed input: 4666.62 toks/s, output: 1198.06 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  93%|█████████▎| 238/256 [00:25<00:00, 118.78it/s, est. speed input: 5245.38 toks/s, output: 1410.50 toks/s]
+(LLMRayActor pid=425159) 
+Processed prompts:  98%|█████████▊| 251/256 [00:25<00:00, 79.35it/s, est. speed input: 5452.61 toks/s, output: 1497.82 toks/s] 
+(LLMRayActor pid=425159) 
+Processed prompts: 100%|██████████| 256/256 [00:26<00:00,  9.82it/s, est. speed input: 5474.42 toks/s, output: 1519.00 toks/s]
+(LLMRayActor pid=425164) 
+Processed prompts:  15%|█▌        | 39/256 [00:25<00:29,  7.25it/s, est. speed input: 1336.35 toks/s, output: 182.84 toks/s]
+Processed prompts:  18%|█▊        | 45/256 [00:25<00:21,  9.95it/s, est. speed input: 1482.33 toks/s, output: 213.47 toks/s]
+Processed prompts:  20%|██        | 52/256 [00:25<00:14, 14.00it/s, est. speed input: 1696.07 toks/s, output: 249.96 toks/s]
+(LLMRayActor pid=425164) 
+Processed prompts:  10%|█         | 26/256 [00:25<01:03,  3.60it/s, est. speed input: 918.31 toks/s, output: 120.04 toks/s]
+Processed prompts:  13%|█▎        | 34/256 [00:25<00:38,  5.84it/s, est. speed input: 1133.54 toks/s, output: 158.71 toks/s]
+(LLMRayActor pid=425164) 
+Processed prompts:  27%|██▋       | 70/256 [00:25<00:06, 27.41it/s, est. speed input: 2135.23 toks/s, output: 346.09 toks/s]
+Processed prompts:  31%|███       | 79/256 [00:26<00:04, 35.65it/s, est. speed input: 2370.95 toks/s, output: 395.33 toks/s]
+(LLMRayActor pid=425164) 
+Processed prompts:  35%|███▌      | 90/256 [00:26<00:03, 47.55it/s, est. speed input: 2635.29 toks/s, output: 456.41 toks/s]
+Processed prompts:  40%|███▉      | 102/256 [00:26<00:02, 61.16it/s, est. speed input: 2859.98 toks/s, output: 524.29 toks/s]
+(LLMRayActor pid=425164) 
+Processed prompts:  25%|██▍       | 63/256 [00:25<00:08, 22.34it/s, est. speed input: 1977.84 toks/s, output: 308.58 toks/s] [repeated 7x across cluster]
+(LLMRayActor pid=425164) 
+Processed prompts:  45%|████▍     | 114/256 [00:26<00:02, 69.35it/s, est. speed input: 3060.31 toks/s, output: 592.60 toks/s]
+Processed prompts:  48%|████▊     | 124/256 [00:26<00:01, 71.98it/s, est. speed input: 3284.29 toks/s, output: 650.05 toks/s]
+(LLMRayActor pid=425164) 
+Processed prompts:  62%|██████▏   | 159/256 [00:26<00:00, 114.56it/s, est. speed input: 4017.09 toks/s, output: 862.82 toks/s]
+Processed prompts:  70%|██████▉   | 179/256 [00:26<00:00, 124.13it/s, est. speed input: 4474.80 toks/s, output: 986.17 toks/s]
+(LLMRayActor pid=425164) 
+Processed prompts:  82%|████████▏ | 211/256 [00:27<00:00, 132.62it/s, est. speed input: 5066.05 toks/s, output: 1191.59 toks/s]
+Processed prompts:  88%|████████▊ | 225/256 [00:27<00:00, 126.30it/s, est. speed input: 5318.39 toks/s, output: 1283.99 toks/s]
+(LLMRayActor pid=425164) 
+Processed prompts: 100%|██████████| 256/256 [00:28<00:00,  9.07it/s, est. speed input: 5732.04 toks/s, output: 1472.70 toks/s] [repeated 3x across cluster]
+(LLMRayActor pid=425163) 
+Processed prompts:  35%|███▌      | 90/256 [00:29<00:03, 49.86it/s, est. speed input: 2562.42 toks/s, output: 378.94 toks/s]
+Processed prompts:  38%|███▊      | 98/256 [00:29<00:02, 56.45it/s, est. speed input: 2746.51 toks/s, output: 417.73 toks/s] [repeated 6x across cluster]
+(LLMRayActor pid=425163) 
+Processed prompts:  52%|█████▏    | 132/256 [00:29<00:01, 84.35it/s, est. speed input: 3454.77 toks/s, output: 589.55 toks/s] [repeated 15x across cluster]
+(LLMRayActor pid=425163) 
+Processed prompts: 100%|██████████| 256/256 [00:31<00:00,  8.02it/s, est. speed input: 5657.66 toks/s, output: 1274.36 toks/s] [repeated 3x across cluster]
+(LLMRayActor pid=425165) 
+Processed prompts:  59%|████��▉    | 151/256 [00:32<00:01, 95.29it/s, est. speed input: 4427.51 toks/s, output: 623.54 toks/s]
+Processed prompts:  66%|██████▌   | 168/256 [00:32<00:00, 108.00it/s, est. speed input: 4764.50 toks/s, output: 706.71 toks/s] [repeated 20x across cluster]
+(LLMRayActor pid=425165) 
+Processed prompts:  79%|███████▊  | 201/256 [00:32<00:00, 113.18it/s, est. speed input: 5314.68 toks/s, output: 872.85 toks/s] [repeated 42x across cluster]
+(LLMRayActor pid=425166) 
+Processed prompts:  99%|█████████▉| 253/256 [00:32<00:00, 72.32it/s, est. speed input: 6219.57 toks/s, output: 1132.26 toks/s] [repeated 12x across cluster]
+(LLMRayActor pid=425160) 
+Processed prompts:   4%|▍         | 10/256 [00:34<05:59,  1.46s/it, est. speed input: 444.93 toks/s, output: 22.33 toks/s]
+Processed prompts:   5%|▌         | 13/256 [00:35<03:34,  1.13it/s, est. speed input: 574.68 toks/s, output: 30.04 toks/s] [repeated 17x across cluster]
+(LLMRayActor pid=425160) 
+Processed prompts:   9%|▉         | 24/256 [00:35<01:03,  3.63it/s, est. speed input: 1040.53 toks/s, output: 59.97 toks/s] [repeated 16x across cluster]
+(LLMRayActor pid=425160) 
+Processed prompts: 100%|██████████| 256/256 [00:38<00:00,  6.57it/s, est. speed input: 6392.40 toks/s, output: 998.23 toks/s] [repeated 4x across cluster]
+(ActorModelRayActor pid=430510) ele.get("min_pixels" 3136
+(ActorModelRayActor pid=430510) ele.get("max_pixels" 1254400
+(ActorModelRayActor pid=430510) ele.get("min_pixels" 3136
+(ActorModelRayActor pid=430510) ele.get("max_pixels" 1254400
+(ActorModelRayActor pid=430510) ele.get("min_pixels" 3136
+(ActorModelRayActor pid=430510) ele.get("max_pixels" 1254400
+(ActorModelRayActor pid=430510) ele.get("min_pixels" 3136
+(ActorModelRayActor pid=430510) ele.get("max_pixels" 1254400
+(ActorModelRayActor pid=430510) ele.get("min_pixels" 3136
+(ActorModelRayActor pid=430510) ele.get("max_pixels" 1254400
+(LLMRayActor pid=425165) INFO 05-28 00:01:14 [worker.py:133] Sleep mode freed 38.53 GiB memory, 17.26 GiB memory is still in use. [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-28 00:01:14 [executor_base.py:208] It took 1.628046 seconds to fall asleep. [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-28 00:01:18 [executor_base.py:219] It took 3.358795 seconds to wake up. [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) ele.get("min_pixels" 3136 [repeated 809x across cluster]
+(ActorModelRayActor pid=430510) ele.get("max_pixels" 1254400 [repeated 809x across cluster]
+(ActorModelRayActor pid=430510) ele.get("min_pixels" 3136 [repeated 712x across cluster]
+(ActorModelRayActor pid=430510) ele.get("max_pixels" 1254400 [repeated 712x across cluster]
+(ActorModelRayActor pid=431336) ele.get("min_pixels" 3136 [repeated 475x across cluster]
+(ActorModelRayActor pid=431336) ele.get("max_pixels" 1254400 [repeated 475x across cluster]
+(LLMRayActor pid=425166) INFO 05-28 00:03:11 [worker.py:133] Sleep mode freed 38.53 GiB memory, 20.04 GiB memory is still in use.
+(LLMRayActor pid=425166) INFO 05-28 00:03:11 [executor_base.py:208] It took 1.397814 seconds to fall asleep.
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   0%|          | 0/128 [00:00<?, ?it/s]
+(LLMRayActor pid=425160) 
+Processed prompts:  72%|███████▏  | 184/256 [00:37<00:00, 105.03it/s, est. speed input: 5319.59 toks/s, output: 662.69 toks/s]
+Processed prompts:  77%|███████▋  | 197/256 [00:37<00:00, 109.21it/s, est. speed input: 5595.06 toks/s, output: 720.10 toks/s] [repeated 5x across cluster]
+(LLMRayActor pid=425160) 
+Processed prompts:  86%|████████▌ | 220/256 [00:38<00:00, 99.16it/s, est. speed input: 5958.07 toks/s, output: 825.37 toks/s]  [repeated 9x across cluster]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   1%|          | 1/128 [00:22<46:57, 22.19s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   2%|▏         | 2/128 [00:24<21:31, 10.25s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   2%|▏         | 3/128 [00:26<14:14,  6.83s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   3%|▎         | 4/128 [00:28<09:46,  4.73s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   4%|▍         | 5/128 [00:29<07:20,  3.58s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   5%|▍         | 6/128 [00:31<05:47,  2.85s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   5%|▌         | 7/128 [00:32<04:49,  2.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   6%|▋         | 8/128 [00:34<04:10,  2.09s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   7%|▋         | 9/128 [00:35<03:50,  1.93s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   8%|▊         | 10/128 [00:37<03:31,  1.79s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   9%|▊         | 11/128 [00:38<03:19,  1.71s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   9%|▉         | 12/128 [00:40<03:09,  1.64s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  10%|█         | 13/128 [00:41<03:03,  1.59s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  11%|█         | 14/128 [00:43<02:56,  1.55s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  12%|█▏        | 15/128 [00:44<02:51,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  12%|█▎        | 16/128 [00:46<02:49,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  13%|█▎        | 17/128 [00:47<02:48,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  14%|█▍        | 18/128 [00:49<02:42,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  15%|█▍        | 19/128 [00:50<02:38,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  16%|█▌        | 20/128 [00:51<02:37,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  16%|█▋        | 21/128 [00:53<02:33,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  17%|█▋        | 22/128 [00:54<02:31,  1.43s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  18%|█▊        | 23/128 [00:56<02:31,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  19%|█▉        | 24/128 [00:57<02:30,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  20%|█▉        | 25/128 [00:59<02:28,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  20%|██        | 26/128 [01:00<02:28,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  21%|██        | 27/128 [01:01<02:24,  1.43s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  22%|██▏       | 28/128 [01:03<02:26,  1.46s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  23%|██▎       | 29/128 [01:05<02:26,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  23%|██▎       | 30/128 [01:06<02:24,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  24%|██▍       | 31/128 [01:08<02:25,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  25%|██▌       | 32/128 [01:09<02:23,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  26%|██▌       | 33/128 [01:10<02:20,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  27%|██▋       | 34/128 [01:12<02:17,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  27%|██▋       | 35/128 [01:13<02:15,  1.46s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  28%|██▊       | 36/128 [01:15<02:16,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  29%|██▉       | 37/128 [01:16<02:16,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  30%|██▉       | 38/128 [01:18<02:14,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  30%|███       | 39/128 [01:19<02:11,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  31%|███▏      | 40/128 [01:21<02:07,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  32%|███▏      | 41/128 [01:22<02:11,  1.51s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  33%|███▎      | 42/128 [01:24<02:10,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  34%|███▎      | 43/128 [01:26<02:11,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  34%|███▍      | 44/128 [01:27<02:10,  1.56s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  35%|███▌      | 45/128 [01:29<02:09,  1.56s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  36%|███▌      | 46/128 [01:30<02:06,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  37%|███▋      | 47/128 [01:32<02:05,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  38%|███▊      | 48/128 [01:33<02:03,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  38%|███▊      | 49/128 [01:35<02:00,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  39%|███▉      | 50/128 [01:36<01:56,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  40%|███▉      | 51/128 [01:38<01:53,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  41%|████      | 52/128 [01:39<01:51,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  41%|████▏     | 53/128 [01:41<01:51,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  42%|████▏     | 54/128 [01:42<01:48,  1.46s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  43%|████▎     | 55/128 [01:43<01:47,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  44%|████▍     | 56/128 [01:45<01:44,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  45%|████▍     | 57/128 [01:46<01:44,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  45%|████▌     | 58/128 [01:48<01:42,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  46%|████▌     | 59/128 [01:49<01:41,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  47%|████▋     | 60/128 [01:51<01:40,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  48%|████▊     | 61/128 [01:52<01:36,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  48%|████▊     | 62/128 [01:54<01:35,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  49%|████▉     | 63/128 [01:55<01:35,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  50%|█████     | 64/128 [01:57<01:35,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  51%|█████     | 65/128 [01:58<01:32,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  52%|█████▏    | 66/128 [02:00<01:30,  1.46s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  52%|█████▏    | 67/128 [02:01<01:28,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  53%|█████▎    | 68/128 [02:02<01:25,  1.43s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  54%|█████▍    | 69/128 [02:04<01:25,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  55%|█████▍    | 70/128 [02:05<01:23,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  55%|█████▌    | 71/128 [02:07<01:23,  1.46s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  56%|█████▋    | 72/128 [02:08<01:21,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  57%|█████▋    | 73/128 [02:10<01:17,  1.41s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  58%|█████▊    | 74/128 [02:11<01:15,  1.40s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  59%|█████▊    | 75/128 [02:12<01:14,  1.40s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  59%|█████▉    | 76/128 [02:14<01:15,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  60%|██████    | 77/128 [02:15<01:14,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  61%|██████    | 78/128 [02:17<01:11,  1.43s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  62%|██████▏   | 79/128 [02:18<01:09,  1.42s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  62%|██████▎   | 80/128 [02:22<01:37,  2.03s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  63%|██████▎   | 81/128 [02:23<01:31,  1.94s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  64%|██████▍   | 82/128 [02:26<01:38,  2.14s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  65%|██████▍   | 83/128 [02:28<01:38,  2.20s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  66%|██████▌   | 84/128 [02:30<01:27,  1.99s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  66%|██████▋   | 85/128 [02:31<01:18,  1.83s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  67%|██████▋   | 86/128 [02:33<01:13,  1.74s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  68%|██████▊   | 87/128 [02:34<01:10,  1.71s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  69%|██████▉   | 88/128 [02:36<01:05,  1.65s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  70%|██████▉   | 89/128 [02:37<01:01,  1.57s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  70%|███████   | 90/128 [02:39<00:58,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  71%|███████   | 91/128 [02:40<00:56,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  72%|███████▏  | 92/128 [02:42<00:53,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  73%|███████▎  | 93/128 [02:43<00:53,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  73%|███████▎  | 94/128 [02:45<00:51,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  74%|███████▍  | 95/128 [02:46<00:49,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  75%|███████▌  | 96/128 [02:48<00:48,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  76%|███████▌  | 97/128 [02:49<00:45,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  77%|███████▋  | 98/128 [02:51<00:43,  1.46s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  77%|███████▋  | 99/128 [02:52<00:42,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  78%|███████▊  | 100/128 [02:54<00:41,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  79%|███████▉  | 101/128 [02:55<00:39,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  80%|███████▉  | 102/128 [02:56<00:37,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  80%|████████  | 103/128 [02:58<00:35,  1.42s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  81%|████████▏ | 104/128 [02:59<00:34,  1.43s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  82%|████████▏ | 105/128 [03:01<00:33,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  83%|████████▎ | 106/128 [03:02<00:32,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  84%|████████▎ | 107/128 [03:04<00:31,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  84%|████████▍ | 108/128 [03:05<00:31,  1.56s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  85%|████████▌ | 109/128 [03:07<00:29,  1.55s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  86%|████████▌ | 110/128 [03:08<00:27,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  87%|████████▋ | 111/128 [03:10<00:26,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  88%|████████▊ | 112/128 [03:12<00:24,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  88%|████████▊ | 113/128 [03:13<00:23,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  89%|████████▉ | 114/128 [03:15<00:21,  1.56s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  90%|████████▉ | 115/128 [03:16<00:20,  1.55s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  91%|█████████ | 116/128 [03:18<00:18,  1.53s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  91%|█████████▏| 117/128 [03:19<00:16,  1.53s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  92%|█████████▏| 118/128 [03:21<00:15,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  93%|█████████▎| 119/128 [03:22<00:13,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  94%|█████████▍| 120/128 [03:24<00:12,  1.53s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  95%|█████████▍| 121/128 [03:25<00:10,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  95%|█████████▌| 122/128 [03:27<00:08,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  96%|█████████▌| 123/128 [03:28<00:07,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  97%|█████████▋| 124/128 [03:30<00:05,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  98%|█████████▊| 125/128 [03:31<00:04,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  98%|█████████▊| 126/128 [03:33<00:03,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  99%|█████████▉| 127/128 [03:34<00:01,  1.55s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience: 100%|██████████| 128/128 [03:36<00:00,  1.52s/it]
+make_experience: 100%|██████████| 128/128 [03:36<00:00,  1.69s/it]
+(ActorModelRayActor pid=430510) ['system\nA conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. The reasoning process and answer are enclosed within <think> </think> and <answer> </answer> tags, respectively, i.e., <think> reasoning process here </think><answer> answer here </answer>\nuser\nWhat is your overall rating of this image’s techniqual quality? Provide a float between 0 and 5, rounded to two decimal places. A rating of 0 represents very poor quality, while 5 represents excellent quality.\nassistant\n<think>The image showcases a dynamic and creative light painting technique, where the light sources are deliberately moved to capture the shapes and patterns in the photograph. The exposure time is long enough to capture the light trails of the light sources, which create a visually appealing and artistic effect. The image appears slightly grainy, which might be due to the long exposure time, and the sharpness is not high, as the image is intentionally designed to give a sense of motion and light. The composition is balanced, with the light trails forming a coherent visual narrative. However, the image lacks detail in the darkness, which might be due to the focus and resolution limitations of the photography process used here. The colors are vibrant, particularly the warm tones of the light, which add to the visual appeal but reduce the image to a primarily aesthetic portrayal rather than a detailed technical one.</think>\n<answer>3.75</answer>']
+(ActorModelRayActor pid=431336) ele.get("min_pixels" 3136 [repeated 47x across cluster]
+(ActorModelRayActor pid=431336) ele.get("max_pixels" 1254400 [repeated 47x across cluster]
+(LLMRayActor pid=425165) INFO 05-28 00:03:11 [worker.py:133] Sleep mode freed 38.49 GiB memory, 21.46 GiB memory is still in use. [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-28 00:03:11 [executor_base.py:208] It took 1.673811 seconds to fall asleep. [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   0%|          | 0/128 [00:00<?, ?it/s]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   0%|          | 0/128 [00:04<?, ?it/s, pg=0.106, rm=0.875, ret=-0.125, glen=158, tlen=843, kl=0.242, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   1%|          | 1/128 [00:04<10:20,  4.89s/it, pg=0.106, rm=0.875, ret=-0.125, glen=158, tlen=843, kl=0.242, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   1%|          | 1/128 [00:07<10:20,  4.89s/it, pg=0.0438, rm=1, ret=-0.0637, glen=162, tlen=796, kl=0.265, act_lr=9.61e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   2%|▏         | 2/128 [00:07<07:04,  3.37s/it, pg=0.0438, rm=1, ret=-0.0637, glen=162, tlen=796, kl=0.265, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   2%|▏         | 2/128 [00:10<07:04,  3.37s/it, pg=0.133, rm=0.75, ret=-0.15, glen=157, tlen=866, kl=0.233, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   2%|▏         | 3/128 [00:10<06:52,  3.30s/it, pg=0.133, rm=0.75, ret=-0.15, glen=157, tlen=866, kl=0.233, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   2%|▏         | 3/128 [00:13<06:52,  3.30s/it, pg=-0.0186, rm=0.875, ret=-0.00113, glen=164, tlen=866, kl=0.248, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   3%|▎         | 4/128 [00:13<06:29,  3.14s/it, pg=-0.0186, rm=0.875, ret=-0.00113, glen=164, tlen=866, kl=0.248, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   3%|▎         | 4/128 [00:15<06:29,  3.14s/it, pg=-0.0727, rm=1.12, ret=0.0546, glen=140, tlen=1.09e+3, kl=0.262, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   4%|▍         | 5/128 [00:15<05:58,  2.92s/it, pg=-0.0727, rm=1.12, ret=0.0546, glen=140, tlen=1.09e+3, kl=0.262, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   4%|▍         | 5/128 [00:17<05:58,  2.92s/it, pg=0.0284, rm=0.688, ret=-0.0482, glen=157, tlen=992, kl=0.275, act_lr=9.61e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   5%|▍         | 6/128 [00:17<05:19,  2.62s/it, pg=0.0284, rm=0.688, ret=-0.0482, glen=157, tlen=992, kl=0.275, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   5%|▍         | 6/128 [00:20<05:19,  2.62s/it, pg=0.00382, rm=1, ret=-0.0218, glen=152, tlen=974, kl=0.261, act_lr=9.61e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   5%|▌         | 7/128 [00:20<05:06,  2.53s/it, pg=0.00382, rm=1, ret=-0.0218, glen=152, tlen=974, kl=0.261, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   5%|▌         | 7/128 [00:31<05:06,  2.53s/it, pg=0.0472, rm=1.06, ret=-0.064, glen=148, tlen=1027.0, kl=0.245, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   6%|▋         | 8/128 [00:31<10:24,  5.20s/it, pg=0.0472, rm=1.06, ret=-0.064, glen=148, tlen=1027.0, kl=0.245, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   6%|▋         | 8/128 [00:33<10:24,  5.20s/it, pg=0.124, rm=0.75, ret=-0.141, glen=150, tlen=969, kl=0.25, act_lr=9.61e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   7%|▋         | 9/128 [00:33<08:24,  4.24s/it, pg=0.124, rm=0.75, ret=-0.141, glen=150, tlen=969, kl=0.25, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   7%|▋         | 9/128 [00:35<08:24,  4.24s/it, pg=0.173, rm=0.688, ret=-0.191, glen=158, tlen=923, kl=0.242, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   8%|▊         | 10/128 [00:35<07:12,  3.66s/it, pg=0.173, rm=0.688, ret=-0.191, glen=158, tlen=923, kl=0.242, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   8%|▊         | 10/128 [00:37<07:12,  3.66s/it, pg=0.0489, rm=1, ret=-0.068, glen=158, tlen=981, kl=0.237, act_lr=9.61e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   9%|▊         | 11/128 [00:37<06:18,  3.24s/it, pg=0.0489, rm=1, ret=-0.068, glen=158, tlen=981, kl=0.237, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   9%|▊         | 11/128 [00:40<06:18,  3.24s/it, pg=0.102, rm=0.812, ret=-0.121, glen=159, tlen=918, kl=0.231, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   9%|▉         | 12/128 [00:40<05:40,  2.94s/it, pg=0.102, rm=0.812, ret=-0.121, glen=159, tlen=918, kl=0.231, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   9%|▉         | 12/128 [00:42<05:40,  2.94s/it, pg=0.0253, rm=0.688, ret=-0.0441, glen=150, tlen=985, kl=0.261, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  10%|█         | 13/128 [00:42<05:06,  2.67s/it, pg=0.0253, rm=0.688, ret=-0.0441, glen=150, tlen=985, kl=0.261, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  10%|█         | 13/128 [00:44<05:06,  2.67s/it, pg=0.197, rm=0.75, ret=-0.213, glen=155, tlen=924, kl=0.216, act_lr=9.61e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  11%|█         | 14/128 [00:44<04:50,  2.55s/it, pg=0.197, rm=0.75, ret=-0.213, glen=155, tlen=924, kl=0.216, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  11%|█         | 14/128 [00:46<04:50,  2.55s/it, pg=0.305, rm=0.812, ret=-0.324, glen=159, tlen=975, kl=0.256, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  12%|█▏        | 15/128 [00:46<04:28,  2.38s/it, pg=0.305, rm=0.812, ret=-0.324, glen=159, tlen=975, kl=0.256, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) [2025-05-28 00:07:54,676] [WARNING] [stage3.py:2139:step] 1 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  12%|█▏        | 15/128 [00:58<04:28,  2.38s/it, pg=0.00498, rm=0.875, ret=-0.0266, glen=157, tlen=854, kl=0.287, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  12%|█▎        | 16/128 [00:58<09:48,  5.25s/it, pg=0.00498, rm=0.875, ret=-0.0266, glen=157, tlen=854, kl=0.287, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  12%|█▎        | 16/128 [01:00<09:48,  5.25s/it, pg=0.0287, rm=1.06, ret=-0.0457, glen=160, tlen=957, kl=0.241, act_lr=9.61e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  13%|█▎        | 17/128 [01:00<07:56,  4.30s/it, pg=0.0287, rm=1.06, ret=-0.0457, glen=160, tlen=957, kl=0.241, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  13%|█▎        | 17/128 [01:02<07:56,  4.30s/it, pg=0.0496, rm=1, ret=-0.0682, glen=150, tlen=845, kl=0.259, act_lr=9.61e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  14%|█▍        | 18/128 [01:02<06:37,  3.62s/it, pg=0.0496, rm=1, ret=-0.0682, glen=150, tlen=845, kl=0.259, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  14%|█▍        | 18/128 [01:04<06:37,  3.62s/it, pg=0.102, rm=1.12, ret=-0.125, glen=168, tlen=852, kl=0.266, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  15%|█▍        | 19/128 [01:04<05:41,  3.13s/it, pg=0.102, rm=1.12, ret=-0.125, glen=168, tlen=852, kl=0.266, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  15%|█▍        | 19/128 [01:06<05:41,  3.13s/it, pg=0.000386, rm=1, ret=-0.0177, glen=145, tlen=1160.75, kl=0.223, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  16%|█▌        | 20/128 [01:06<05:15,  2.92s/it, pg=0.000386, rm=1, ret=-0.0177, glen=145, tlen=1160.75, kl=0.223, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  16%|█▌        | 20/128 [01:09<05:15,  2.92s/it, pg=0.117, rm=0.812, ret=-0.136, glen=150, tlen=976, kl=0.248, act_lr=9.61e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  16%|█▋        | 21/128 [01:09<04:57,  2.78s/it, pg=0.117, rm=0.812, ret=-0.136, glen=150, tlen=976, kl=0.248, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  16%|█▋        | 21/128 [01:11<04:57,  2.78s/it, pg=-0.151, rm=0.875, ret=0.132, glen=155, tlen=850, kl=0.252, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  17%|█▋        | 22/128 [01:11<04:30,  2.55s/it, pg=-0.151, rm=0.875, ret=0.132, glen=155, tlen=850, kl=0.252, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  17%|█▋        | 22/128 [01:13<04:30,  2.55s/it, pg=0.0152, rm=0.938, ret=-0.0343, glen=158, tlen=797, kl=0.262, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  18%|█▊        | 23/128 [01:13<04:14,  2.43s/it, pg=0.0152, rm=0.938, ret=-0.0343, glen=158, tlen=797, kl=0.262, act_lr=9.61e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  18%|█▊        | 23/128 [01:25<04:14,  2.43s/it, pg=0.131, rm=0.812, ret=-0.151, glen=161, tlen=954, kl=0.253, act_lr=9.6e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  19%|█▉        | 24/128 [01:25<09:15,  5.34s/it, pg=0.131, rm=0.812, ret=-0.151, glen=161, tlen=954, kl=0.253, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  19%|█▉        | 24/128 [01:28<09:15,  5.34s/it, pg=0.072, rm=0.75, ret=-0.0887, glen=143, tlen=969, kl=0.251, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  20%|█▉        | 25/128 [01:28<07:38,  4.45s/it, pg=0.072, rm=0.75, ret=-0.0887, glen=143, tlen=969, kl=0.251, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  20%|█▉        | 25/128 [01:29<07:38,  4.45s/it, pg=0.00993, rm=1, ret=-0.0287, glen=154, tlen=921, kl=0.237, act_lr=9.6e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  20%|██        | 26/128 [01:29<06:15,  3.68s/it, pg=0.00993, rm=1, ret=-0.0287, glen=154, tlen=921, kl=0.237, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  20%|██        | 26/128 [01:32<06:15,  3.68s/it, pg=0.0892, rm=0.875, ret=-0.106, glen=160, tlen=965, kl=0.236, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  21%|██        | 27/128 [01:32<05:31,  3.28s/it, pg=0.0892, rm=0.875, ret=-0.106, glen=160, tlen=965, kl=0.236, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  21%|██        | 27/128 [01:34<05:31,  3.28s/it, pg=0.0466, rm=1.06, ret=-0.0632, glen=143, tlen=1.02e+3, kl=0.248, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  22%|██▏       | 28/128 [01:34<04:58,  2.98s/it, pg=0.0466, rm=1.06, ret=-0.0632, glen=143, tlen=1.02e+3, kl=0.248, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  22%|██▏       | 28/128 [01:36<04:58,  2.98s/it, pg=-0.209, rm=1.06, ret=0.19, glen=154, tlen=985, kl=0.24, act_lr=9.6e-7]        
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  23%|██▎       | 29/128 [01:36<04:27,  2.70s/it, pg=-0.209, rm=1.06, ret=0.19, glen=154, tlen=985, kl=0.24, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  23%|██▎       | 29/128 [01:38<04:27,  2.70s/it, pg=0.0404, rm=0.875, ret=-0.0577, glen=149, tlen=874, kl=0.256, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  23%|██▎       | 30/128 [01:38<04:11,  2.56s/it, pg=0.0404, rm=0.875, ret=-0.0577, glen=149, tlen=874, kl=0.256, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  23%|██▎       | 30/128 [01:40<04:11,  2.56s/it, pg=0.013, rm=1.19, ret=-0.0342, glen=160, tlen=810, kl=0.288, act_lr=9.6e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  24%|██▍       | 31/128 [01:40<03:51,  2.39s/it, pg=0.013, rm=1.19, ret=-0.0342, glen=160, tlen=810, kl=0.288, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  24%|██▍       | 31/128 [01:52<03:51,  2.39s/it, pg=0.114, rm=0.812, ret=-0.134, glen=158, tlen=912, kl=0.262, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  25%|██▌       | 32/128 [01:52<08:06,  5.07s/it, pg=0.114, rm=0.812, ret=-0.134, glen=158, tlen=912, kl=0.262, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  25%|██▌       | 32/128 [01:54<08:06,  5.07s/it, pg=-0.00922, rm=0.875, ret=-0.0102, glen=162, tlen=925, kl=0.255, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  26%|██▌       | 33/128 [01:54<06:45,  4.27s/it, pg=-0.00922, rm=0.875, ret=-0.0102, glen=162, tlen=925, kl=0.255, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  26%|██▌       | 33/128 [01:56<06:45,  4.27s/it, pg=0.17, rm=0.938, ret=-0.189, glen=160, tlen=840, kl=0.259, act_lr=9.6e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  27%|██▋       | 34/128 [01:56<05:40,  3.62s/it, pg=0.17, rm=0.938, ret=-0.189, glen=160, tlen=840, kl=0.259, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  27%|██▋       | 34/128 [01:58<05:40,  3.62s/it, pg=-0.0273, rm=0.938, ret=0.00634, glen=158, tlen=924, kl=0.269, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  27%|██▋       | 35/128 [01:58<05:00,  3.23s/it, pg=-0.0273, rm=0.938, ret=0.00634, glen=158, tlen=924, kl=0.269, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  27%|██▋       | 35/128 [02:01<05:00,  3.23s/it, pg=-0.00986, rm=1.19, ret=-0.0105, glen=160, tlen=787, kl=0.261, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  28%|██▊       | 36/128 [02:01<04:35,  2.99s/it, pg=-0.00986, rm=1.19, ret=-0.0105, glen=160, tlen=787, kl=0.261, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  28%|██▊       | 36/128 [02:03<04:35,  2.99s/it, pg=-0.0327, rm=0.688, ret=0.0124, glen=164, tlen=918, kl=0.247, act_lr=9.6e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  29%|██▉       | 37/128 [02:03<04:01,  2.65s/it, pg=-0.0327, rm=0.688, ret=0.0124, glen=164, tlen=918, kl=0.247, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  29%|██▉       | 37/128 [02:05<04:01,  2.65s/it, pg=0.129, rm=0.812, ret=-0.149, glen=161, tlen=954, kl=0.244, act_lr=9.6e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  30%|██▉       | 38/128 [02:05<03:41,  2.47s/it, pg=0.129, rm=0.812, ret=-0.149, glen=161, tlen=954, kl=0.244, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  30%|██▉       | 38/128 [02:07<03:41,  2.47s/it, pg=0.166, rm=0.688, ret=-0.186, glen=151, tlen=960, kl=0.248, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  30%|███       | 39/128 [02:07<03:33,  2.40s/it, pg=0.166, rm=0.688, ret=-0.186, glen=151, tlen=960, kl=0.248, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  30%|███       | 39/128 [02:17<03:33,  2.40s/it, pg=0.147, rm=0.938, ret=-0.167, glen=144, tlen=1.02e+3, kl=0.248, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  31%|███▏      | 40/128 [02:17<06:58,  4.76s/it, pg=0.147, rm=0.938, ret=-0.167, glen=144, tlen=1.02e+3, kl=0.248, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  31%|███▏      | 40/128 [02:19<06:58,  4.76s/it, pg=0.0265, rm=0.812, ret=-0.0441, glen=155, tlen=846, kl=0.252, act_lr=9.6e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  32%|███▏      | 41/128 [02:19<05:40,  3.91s/it, pg=0.0265, rm=0.812, ret=-0.0441, glen=155, tlen=846, kl=0.252, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  32%|███▏      | 41/128 [02:21<05:40,  3.91s/it, pg=-0.0935, rm=1.12, ret=0.0777, glen=146, tlen=1.02e+3, kl=0.236, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  33%|███▎      | 42/128 [02:21<04:52,  3.40s/it, pg=-0.0935, rm=1.12, ret=0.0777, glen=146, tlen=1.02e+3, kl=0.236, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  33%|███▎      | 42/128 [02:23<04:52,  3.40s/it, pg=0.176, rm=1.06, ret=-0.194, glen=153, tlen=951, kl=0.245, act_lr=9.6e-7]      
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  34%|███▎      | 43/128 [02:23<04:13,  2.98s/it, pg=0.176, rm=1.06, ret=-0.194, glen=153, tlen=951, kl=0.245, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  34%|███▎      | 43/128 [02:25<04:13,  2.98s/it, pg=0.223, rm=0.875, ret=-0.242, glen=160, tlen=795, kl=0.26, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  34%|███▍      | 44/128 [02:25<03:45,  2.68s/it, pg=0.223, rm=0.875, ret=-0.242, glen=160, tlen=795, kl=0.26, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  34%|███▍      | 44/128 [02:28<03:45,  2.68s/it, pg=0.232, rm=1.06, ret=-0.252, glen=158, tlen=1043.25, kl=0.25, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  35%|███▌      | 45/128 [02:28<03:36,  2.61s/it, pg=0.232, rm=1.06, ret=-0.252, glen=158, tlen=1043.25, kl=0.25, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  35%|███▌      | 45/128 [02:30<03:36,  2.61s/it, pg=-0.0696, rm=0.938, ret=0.0521, glen=145, tlen=834, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  36%|███▌      | 46/128 [02:30<03:19,  2.44s/it, pg=-0.0696, rm=0.938, ret=0.0521, glen=145, tlen=834, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  36%|███▌      | 46/128 [02:32<03:19,  2.44s/it, pg=0.24, rm=1.06, ret=-0.263, glen=176, tlen=932, kl=0.262, act_lr=9.6e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  37%|███▋      | 47/128 [02:32<03:11,  2.37s/it, pg=0.24, rm=1.06, ret=-0.263, glen=176, tlen=932, kl=0.262, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  37%|███▋      | 47/128 [02:43<03:11,  2.37s/it, pg=-0.0429, rm=1, ret=0.0247, glen=158, tlen=860, kl=0.249, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  38%|███▊      | 48/128 [02:43<06:22,  4.78s/it, pg=-0.0429, rm=1, ret=0.0247, glen=158, tlen=860, kl=0.249, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  38%|███▊      | 48/128 [02:44<06:22,  4.78s/it, pg=-0.12, rm=1.06, ret=0.102, glen=162, tlen=807, kl=0.251, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  38%|███▊      | 49/128 [02:44<05:10,  3.93s/it, pg=-0.12, rm=1.06, ret=0.102, glen=162, tlen=807, kl=0.251, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  38%|███▊      | 49/128 [02:47<05:10,  3.93s/it, pg=-0.145, rm=1, ret=0.128, glen=159, tlen=794, kl=0.258, act_lr=9.6e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  39%|███▉      | 50/128 [02:47<04:25,  3.40s/it, pg=-0.145, rm=1, ret=0.128, glen=159, tlen=794, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  39%|███▉      | 50/128 [02:49<04:25,  3.40s/it, pg=-0.191, rm=1.06, ret=0.172, glen=156, tlen=978, kl=0.241, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  40%|███▉      | 51/128 [02:49<03:56,  3.07s/it, pg=-0.191, rm=1.06, ret=0.172, glen=156, tlen=978, kl=0.241, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  40%|███▉      | 51/128 [02:51<03:56,  3.07s/it, pg=-0.0513, rm=1, ret=0.0333, glen=164, tlen=995, kl=0.236, act_lr=9.6e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  41%|████      | 52/128 [02:51<03:30,  2.77s/it, pg=-0.0513, rm=1, ret=0.0333, glen=164, tlen=995, kl=0.236, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  41%|████      | 52/128 [02:53<03:30,  2.77s/it, pg=-0.0318, rm=0.688, ret=0.0129, glen=161, tlen=916, kl=0.249, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  41%|████▏     | 53/128 [02:53<03:12,  2.57s/it, pg=-0.0318, rm=0.688, ret=0.0129, glen=161, tlen=916, kl=0.249, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  41%|████▏     | 53/128 [02:55<03:12,  2.57s/it, pg=-0.0611, rm=1, ret=0.0439, glen=144, tlen=829, kl=0.258, act_lr=9.6e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  42%|████▏     | 54/128 [02:55<02:58,  2.41s/it, pg=-0.0611, rm=1, ret=0.0439, glen=144, tlen=829, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  42%|████▏     | 54/128 [02:57<02:58,  2.41s/it, pg=0.0407, rm=0.875, ret=-0.0596, glen=155, tlen=880, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  43%|████▎     | 55/128 [02:57<02:52,  2.36s/it, pg=0.0407, rm=0.875, ret=-0.0596, glen=155, tlen=880, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  43%|████▎     | 55/128 [03:07<02:52,  2.36s/it, pg=0.324, rm=0.938, ret=-0.339, glen=151, tlen=858, kl=0.266, act_lr=9.6e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  44%|████▍     | 56/128 [03:07<05:36,  4.67s/it, pg=0.324, rm=0.938, ret=-0.339, glen=151, tlen=858, kl=0.266, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  44%|████▍     | 56/128 [03:10<05:36,  4.67s/it, pg=0.134, rm=0.938, ret=-0.153, glen=162, tlen=857, kl=0.255, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  45%|████▍     | 57/128 [03:10<04:44,  4.00s/it, pg=0.134, rm=0.938, ret=-0.153, glen=162, tlen=857, kl=0.255, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  45%|████▍     | 57/128 [03:12<04:44,  4.00s/it, pg=0.0526, rm=0.875, ret=-0.0694, glen=165, tlen=854, kl=0.231, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  45%|████▌     | 58/128 [03:12<03:55,  3.36s/it, pg=0.0526, rm=0.875, ret=-0.0694, glen=165, tlen=854, kl=0.231, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  45%|████▌     | 58/128 [03:14<03:55,  3.36s/it, pg=0.0707, rm=0.75, ret=-0.0905, glen=165, tlen=903, kl=0.255, act_lr=9.6e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  46%|████▌     | 59/128 [03:14<03:26,  2.99s/it, pg=0.0707, rm=0.75, ret=-0.0905, glen=165, tlen=903, kl=0.255, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  46%|████▌     | 59/128 [03:16<03:26,  2.99s/it, pg=0.0983, rm=0.875, ret=-0.115, glen=140, tlen=1.1e+3, kl=0.252, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  47%|████▋     | 60/128 [03:16<03:10,  2.80s/it, pg=0.0983, rm=0.875, ret=-0.115, glen=140, tlen=1.1e+3, kl=0.252, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  47%|████▋     | 60/128 [03:18<03:10,  2.80s/it, pg=-0.0206, rm=0.812, ret=0.00314, glen=160, tlen=1e+3, kl=0.235, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  48%|████▊     | 61/128 [03:18<02:52,  2.58s/it, pg=-0.0206, rm=0.812, ret=0.00314, glen=160, tlen=1e+3, kl=0.235, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  48%|████▊     | 61/128 [03:21<02:52,  2.58s/it, pg=-0.341, rm=1.19, ret=0.324, glen=153, tlen=1.03e+3, kl=0.249, act_lr=9.6e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  48%|████▊     | 62/128 [03:21<02:42,  2.47s/it, pg=-0.341, rm=1.19, ret=0.324, glen=153, tlen=1.03e+3, kl=0.249, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  48%|████▊     | 62/128 [03:23<02:42,  2.47s/it, pg=0.198, rm=0.562, ret=-0.217, glen=147, tlen=969, kl=0.27, act_lr=9.6e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  49%|████▉     | 63/128 [03:23<02:36,  2.40s/it, pg=0.198, rm=0.562, ret=-0.217, glen=147, tlen=969, kl=0.27, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  49%|████▉     | 63/128 [03:33<02:36,  2.40s/it, pg=0.106, rm=0.812, ret=-0.124, glen=160, tlen=862, kl=0.242, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  50%|█████     | 64/128 [03:33<05:08,  4.82s/it, pg=0.106, rm=0.812, ret=-0.124, glen=160, tlen=862, kl=0.242, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  50%|█████     | 64/128 [03:36<05:08,  4.82s/it, pg=-0.377, rm=0.938, ret=0.359, glen=145, tlen=965, kl=0.235, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  51%|█████     | 65/128 [03:36<04:16,  4.08s/it, pg=-0.377, rm=0.938, ret=0.359, glen=145, tlen=965, kl=0.235, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  51%|█████     | 65/128 [03:37<04:16,  4.08s/it, pg=0.183, rm=0.875, ret=-0.201, glen=138, tlen=890, kl=0.265, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  52%|█████▏    | 66/128 [03:37<03:31,  3.41s/it, pg=0.183, rm=0.875, ret=-0.201, glen=138, tlen=890, kl=0.265, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  52%|█████▏    | 66/128 [03:40<03:31,  3.41s/it, pg=0.275, rm=0.812, ret=-0.292, glen=159, tlen=964, kl=0.259, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  52%|█████▏    | 67/128 [03:40<03:09,  3.10s/it, pg=0.275, rm=0.812, ret=-0.292, glen=159, tlen=964, kl=0.259, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  52%|█████▏    | 67/128 [03:42<03:09,  3.10s/it, pg=-0.0716, rm=0.875, ret=0.0533, glen=155, tlen=804, kl=0.235, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  53%|█████▎    | 68/128 [03:42<02:51,  2.86s/it, pg=-0.0716, rm=0.875, ret=0.0533, glen=155, tlen=804, kl=0.235, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  53%|█████▎    | 68/128 [03:44<02:51,  2.86s/it, pg=0.374, rm=1, ret=-0.391, glen=156, tlen=796, kl=0.254, act_lr=9.6e-7]      
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  54%|█████▍    | 69/128 [03:44<02:36,  2.65s/it, pg=0.374, rm=1, ret=-0.391, glen=156, tlen=796, kl=0.254, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  54%|█████▍    | 69/128 [03:46<02:36,  2.65s/it, pg=0.0096, rm=1.06, ret=-0.0278, glen=156, tlen=891, kl=0.241, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  55%|█████▍    | 70/128 [03:46<02:20,  2.42s/it, pg=0.0096, rm=1.06, ret=-0.0278, glen=156, tlen=891, kl=0.241, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  55%|█████▍    | 70/128 [03:49<02:20,  2.42s/it, pg=-0.263, rm=1.12, ret=0.244, glen=156, tlen=932, kl=0.254, act_lr=9.6e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  55%|█████▌    | 71/128 [03:49<02:18,  2.44s/it, pg=-0.263, rm=1.12, ret=0.244, glen=156, tlen=932, kl=0.254, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  55%|█████▌    | 71/128 [03:59<02:18,  2.44s/it, pg=-0.0409, rm=0.938, ret=0.021, glen=160, tlen=966, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  56%|█████▋    | 72/128 [03:59<04:22,  4.68s/it, pg=-0.0409, rm=0.938, ret=0.021, glen=160, tlen=966, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  56%|█████▋    | 72/128 [04:01<04:22,  4.68s/it, pg=0.0113, rm=1.19, ret=-0.0311, glen=146, tlen=784, kl=0.296, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  57%|█████▋    | 73/128 [04:01<03:34,  3.91s/it, pg=0.0113, rm=1.19, ret=-0.0311, glen=146, tlen=784, kl=0.296, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  57%|█████▋    | 73/128 [04:02<03:34,  3.91s/it, pg=0.0331, rm=0.812, ret=-0.0518, glen=149, tlen=901, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  58%|█████▊    | 74/128 [04:02<02:57,  3.29s/it, pg=0.0331, rm=0.812, ret=-0.0518, glen=149, tlen=901, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  58%|█████▊    | 74/128 [04:05<02:57,  3.29s/it, pg=0.00468, rm=0.875, ret=-0.0209, glen=153, tlen=888, kl=0.239, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  59%|█████▊    | 75/128 [04:05<02:35,  2.93s/it, pg=0.00468, rm=0.875, ret=-0.0209, glen=153, tlen=888, kl=0.239, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  59%|█████▊    | 75/128 [04:07<02:35,  2.93s/it, pg=-0.045, rm=1.12, ret=0.0263, glen=157, tlen=843, kl=0.26, act_lr=9.6e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  59%|█████▉    | 76/128 [04:07<02:18,  2.66s/it, pg=-0.045, rm=1.12, ret=0.0263, glen=157, tlen=843, kl=0.26, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  59%|█████▉    | 76/128 [04:09<02:18,  2.66s/it, pg=-0.0742, rm=0.812, ret=0.0535, glen=162, tlen=810, kl=0.274, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  60%|██████    | 77/128 [04:09<02:05,  2.47s/it, pg=-0.0742, rm=0.812, ret=0.0535, glen=162, tlen=810, kl=0.274, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  60%|██████    | 77/128 [04:11<02:05,  2.47s/it, pg=0.108, rm=0.812, ret=-0.127, glen=163, tlen=865, kl=0.252, act_lr=9.6e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  61%|██████    | 78/128 [04:11<01:57,  2.35s/it, pg=0.108, rm=0.812, ret=-0.127, glen=163, tlen=865, kl=0.252, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  61%|██████    | 78/128 [04:13<01:57,  2.35s/it, pg=-0.233, rm=1.19, ret=0.216, glen=166, tlen=845, kl=0.239, act_lr=9.6e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  62%|██████▏   | 79/128 [04:13<01:54,  2.34s/it, pg=-0.233, rm=1.19, ret=0.216, glen=166, tlen=845, kl=0.239, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  62%|██████▏   | 79/128 [04:23<01:54,  2.34s/it, pg=-0.12, rm=1.06, ret=0.105, glen=152, tlen=928, kl=0.256, act_lr=9.6e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  62%|██████▎   | 80/128 [04:23<03:40,  4.60s/it, pg=-0.12, rm=1.06, ret=0.105, glen=152, tlen=928, kl=0.256, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  62%|██████▎   | 80/128 [04:25<03:40,  4.60s/it, pg=-0.0265, rm=1.12, ret=0.00835, glen=155, tlen=841, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  63%|██████▎   | 81/128 [04:25<03:01,  3.86s/it, pg=-0.0265, rm=1.12, ret=0.00835, glen=155, tlen=841, kl=0.258, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  63%|██████▎   | 81/128 [04:27<03:01,  3.86s/it, pg=-0.0483, rm=1, ret=0.0319, glen=150, tlen=964, kl=0.252, act_lr=9.6e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  64%|██████▍   | 82/128 [04:27<02:36,  3.39s/it, pg=-0.0483, rm=1, ret=0.0319, glen=150, tlen=964, kl=0.252, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  64%|██████▍   | 82/128 [04:29<02:36,  3.39s/it, pg=0.108, rm=0.938, ret=-0.126, glen=161, tlen=857, kl=0.26, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  65%|██████▍   | 83/128 [04:29<02:12,  2.94s/it, pg=0.108, rm=0.938, ret=-0.126, glen=161, tlen=857, kl=0.26, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  65%|██████▍   | 83/128 [04:31<02:12,  2.94s/it, pg=-0.0743, rm=1.06, ret=0.0588, glen=143, tlen=966, kl=0.222, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  66%|██████▌   | 84/128 [04:31<01:58,  2.69s/it, pg=-0.0743, rm=1.06, ret=0.0588, glen=143, tlen=966, kl=0.222, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  66%|██████▌   | 84/128 [04:33<01:58,  2.69s/it, pg=0.00442, rm=0.875, ret=-0.0191, glen=155, tlen=908, kl=0.235, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  66%|██████▋   | 85/128 [04:33<01:45,  2.45s/it, pg=0.00442, rm=0.875, ret=-0.0191, glen=155, tlen=908, kl=0.235, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  66%|██████▋   | 85/128 [04:35<01:45,  2.45s/it, pg=-0.0262, rm=1, ret=0.00543, glen=146, tlen=831, kl=0.285, act_lr=9.6e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  67%|██████▋   | 86/128 [04:35<01:40,  2.38s/it, pg=-0.0262, rm=1, ret=0.00543, glen=146, tlen=831, kl=0.285, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  67%|██████▋   | 86/128 [04:38<01:40,  2.38s/it, pg=-0.18, rm=0.938, ret=0.162, glen=160, tlen=869, kl=0.246, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  68%|██████▊   | 87/128 [04:38<01:39,  2.43s/it, pg=-0.18, rm=0.938, ret=0.162, glen=160, tlen=869, kl=0.246, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  68%|██████▊   | 87/128 [04:47<01:39,  2.43s/it, pg=-0.0118, rm=0.875, ret=-0.00419, glen=160, tlen=913, kl=0.23, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  69%|██████▉   | 88/128 [04:47<03:00,  4.50s/it, pg=-0.0118, rm=0.875, ret=-0.00419, glen=160, tlen=913, kl=0.23, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  69%|██████▉   | 88/128 [04:49<03:00,  4.50s/it, pg=-0.189, rm=0.75, ret=0.174, glen=142, tlen=964, kl=0.252, act_lr=9.6e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  70%|██████▉   | 89/128 [04:49<02:27,  3.77s/it, pg=-0.189, rm=0.75, ret=0.174, glen=142, tlen=964, kl=0.252, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  70%|██████▉   | 89/128 [04:52<02:27,  3.77s/it, pg=0.05, rm=0.875, ret=-0.0691, glen=139, tlen=828, kl=0.271, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  70%|███████   | 90/128 [04:52<02:06,  3.34s/it, pg=0.05, rm=0.875, ret=-0.0691, glen=139, tlen=828, kl=0.271, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  70%|███████   | 90/128 [04:54<02:06,  3.34s/it, pg=0.00247, rm=1, ret=-0.0167, glen=134, tlen=1.15e+3, kl=0.233, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  71%|███████   | 91/128 [04:54<01:49,  2.96s/it, pg=0.00247, rm=1, ret=-0.0167, glen=134, tlen=1.15e+3, kl=0.233, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  71%|███████   | 91/128 [04:56<01:49,  2.96s/it, pg=0.0129, rm=1, ret=-0.0321, glen=157, tlen=852, kl=0.269, act_lr=9.6e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  72%|███████▏  | 92/128 [04:56<01:34,  2.62s/it, pg=0.0129, rm=1, ret=-0.0321, glen=157, tlen=852, kl=0.269, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  72%|███████▏  | 92/128 [04:58<01:34,  2.62s/it, pg=0.0129, rm=1, ret=-0.029, glen=148, tlen=912, kl=0.25, act_lr=9.6e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  73%|███████▎  | 93/128 [04:58<01:29,  2.54s/it, pg=0.0129, rm=1, ret=-0.029, glen=148, tlen=912, kl=0.25, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  73%|███████▎  | 93/128 [05:00<01:29,  2.54s/it, pg=-0.0949, rm=0.812, ret=0.0786, glen=154, tlen=914, kl=0.229, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  73%|███████▎  | 94/128 [05:00<01:22,  2.41s/it, pg=-0.0949, rm=0.812, ret=0.0786, glen=154, tlen=914, kl=0.229, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  73%|███████▎  | 94/128 [05:02<01:22,  2.41s/it, pg=0.0636, rm=0.938, ret=-0.084, glen=148, tlen=900, kl=0.275, act_lr=9.6e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  74%|███████▍  | 95/128 [05:02<01:16,  2.31s/it, pg=0.0636, rm=0.938, ret=-0.084, glen=148, tlen=900, kl=0.275, act_lr=9.6e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  74%|███████▍  | 95/128 [05:12<01:16,  2.31s/it, pg=0.0336, rm=0.812, ret=-0.0508, glen=154, tlen=907, kl=0.242, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  75%|███████▌  | 96/128 [05:12<02:21,  4.42s/it, pg=0.0336, rm=0.812, ret=-0.0508, glen=154, tlen=907, kl=0.242, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  75%|███████▌  | 96/128 [05:13<02:21,  4.42s/it, pg=0.0501, rm=0.875, ret=-0.069, glen=144, tlen=833, kl=0.261, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  76%|███████▌  | 97/128 [05:13<01:53,  3.66s/it, pg=0.0501, rm=0.875, ret=-0.069, glen=144, tlen=833, kl=0.261, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  76%|███████▌  | 97/128 [05:16<01:53,  3.66s/it, pg=0.0598, rm=1.06, ret=-0.081, glen=167, tlen=965, kl=0.262, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  77%|███████▋  | 98/128 [05:16<01:38,  3.29s/it, pg=0.0598, rm=1.06, ret=-0.081, glen=167, tlen=965, kl=0.262, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  77%|███████▋  | 98/128 [05:18<01:38,  3.29s/it, pg=0.196, rm=0.75, ret=-0.211, glen=144, tlen=788, kl=0.234, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  77%|███████▋  | 99/128 [05:18<01:22,  2.86s/it, pg=0.196, rm=0.75, ret=-0.211, glen=144, tlen=788, kl=0.234, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  77%|███████▋  | 99/128 [05:19<01:22,  2.86s/it, pg=0.00373, rm=0.875, ret=-0.0231, glen=155, tlen=853, kl=0.266, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  78%|███████▊  | 100/128 [05:19<01:11,  2.55s/it, pg=0.00373, rm=0.875, ret=-0.0231, glen=155, tlen=853, kl=0.266, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  78%|███████▊  | 100/128 [05:22<01:11,  2.55s/it, pg=-0.014, rm=1.19, ret=-0.0045, glen=149, tlen=917, kl=0.263, act_lr=9.59e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  79%|███████▉  | 101/128 [05:22<01:05,  2.41s/it, pg=-0.014, rm=1.19, ret=-0.0045, glen=149, tlen=917, kl=0.263, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  79%|███████▉  | 101/128 [05:24<01:05,  2.41s/it, pg=0.104, rm=0.812, ret=-0.122, glen=159, tlen=861, kl=0.228, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  80%|███████▉  | 102/128 [05:24<00:59,  2.28s/it, pg=0.104, rm=0.812, ret=-0.122, glen=159, tlen=861, kl=0.228, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  80%|███████▉  | 102/128 [05:25<00:59,  2.28s/it, pg=0.0393, rm=1.06, ret=-0.0576, glen=150, tlen=839, kl=0.256, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  80%|████████  | 103/128 [05:25<00:53,  2.15s/it, pg=0.0393, rm=1.06, ret=-0.0576, glen=150, tlen=839, kl=0.256, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  80%|████████  | 103/128 [05:35<00:53,  2.15s/it, pg=-0.0337, rm=0.938, ret=0.0163, glen=145, tlen=978, kl=0.247, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  81%|████████▏ | 104/128 [05:35<01:43,  4.30s/it, pg=-0.0337, rm=0.938, ret=0.0163, glen=145, tlen=978, kl=0.247, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  81%|████████▏ | 104/128 [05:37<01:43,  4.30s/it, pg=0.0496, rm=0.938, ret=-0.0662, glen=149, tlen=958, kl=0.245, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  82%|████████▏ | 105/128 [05:37<01:25,  3.70s/it, pg=0.0496, rm=0.938, ret=-0.0662, glen=149, tlen=958, kl=0.245, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  82%|████████▏ | 105/128 [05:39<01:25,  3.70s/it, pg=0.0324, rm=1, ret=-0.0503, glen=161, tlen=852, kl=0.244, act_lr=9.59e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  83%|████████▎ | 106/128 [05:39<01:10,  3.23s/it, pg=0.0324, rm=1, ret=-0.0503, glen=161, tlen=852, kl=0.244, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  83%|████████▎ | 106/128 [05:41<01:10,  3.23s/it, pg=-0.0421, rm=1, ret=0.0226, glen=147, tlen=910, kl=0.278, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  84%|████████▎ | 107/128 [05:41<01:00,  2.88s/it, pg=-0.0421, rm=1, ret=0.0226, glen=147, tlen=910, kl=0.278, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  84%|████████▎ | 107/128 [05:43<01:00,  2.88s/it, pg=0.0904, rm=0.625, ret=-0.107, glen=157, tlen=929, kl=0.236, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  84%|████████▍ | 108/128 [05:43<00:51,  2.57s/it, pg=0.0904, rm=0.625, ret=-0.107, glen=157, tlen=929, kl=0.236, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  84%|████████▍ | 108/128 [05:45<00:51,  2.57s/it, pg=-0.0483, rm=0.938, ret=0.0303, glen=157, tlen=842, kl=0.245, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  85%|████████▌ | 109/128 [05:45<00:45,  2.39s/it, pg=-0.0483, rm=0.938, ret=0.0303, glen=157, tlen=842, kl=0.245, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  85%|████████▌ | 109/128 [05:47<00:45,  2.39s/it, pg=-0.185, rm=1, ret=0.167, glen=157, tlen=836, kl=0.272, act_lr=9.59e-7]      
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  86%|████████▌ | 110/128 [05:47<00:41,  2.31s/it, pg=-0.185, rm=1, ret=0.167, glen=157, tlen=836, kl=0.272, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  86%|████████▌ | 110/128 [05:49<00:41,  2.31s/it, pg=0.0878, rm=0.875, ret=-0.106, glen=163, tlen=849, kl=0.242, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  87%|████████▋ | 111/128 [05:49<00:36,  2.17s/it, pg=0.0878, rm=0.875, ret=-0.106, glen=163, tlen=849, kl=0.242, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  87%|████████▋ | 111/128 [05:59<00:36,  2.17s/it, pg=-0.0321, rm=1, ret=0.0156, glen=158, tlen=878, kl=0.243, act_lr=9.59e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  88%|████████▊ | 112/128 [05:59<01:12,  4.51s/it, pg=-0.0321, rm=1, ret=0.0156, glen=158, tlen=878, kl=0.243, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  88%|████████▊ | 112/128 [06:01<01:12,  4.51s/it, pg=0.187, rm=1.06, ret=-0.202, glen=153, tlen=821, kl=0.245, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  88%|████████▊ | 113/128 [06:01<00:56,  3.78s/it, pg=0.187, rm=1.06, ret=-0.202, glen=153, tlen=821, kl=0.245, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  88%|████████▊ | 113/128 [06:03<00:56,  3.78s/it, pg=0.0919, rm=0.625, ret=-0.112, glen=154, tlen=926, kl=0.27, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  89%|████████▉ | 114/128 [06:03<00:45,  3.25s/it, pg=0.0919, rm=0.625, ret=-0.112, glen=154, tlen=926, kl=0.27, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  89%|████████▉ | 114/128 [06:05<00:45,  3.25s/it, pg=0.00193, rm=1.12, ret=-0.0205, glen=168, tlen=807, kl=0.238, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  90%|████████▉ | 115/128 [06:05<00:37,  2.92s/it, pg=0.00193, rm=1.12, ret=-0.0205, glen=168, tlen=807, kl=0.238, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  90%|████████▉ | 115/128 [06:07<00:37,  2.92s/it, pg=0.0014, rm=1.12, ret=-0.0177, glen=153, tlen=793, kl=0.242, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  91%|█████████ | 116/128 [06:07<00:32,  2.67s/it, pg=0.0014, rm=1.12, ret=-0.0177, glen=153, tlen=793, kl=0.242, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  91%|█████████ | 116/128 [06:09<00:32,  2.67s/it, pg=-0.152, rm=1, ret=0.136, glen=157, tlen=871, kl=0.242, act_lr=9.59e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  91%|█████████▏| 117/128 [06:09<00:27,  2.51s/it, pg=-0.152, rm=1, ret=0.136, glen=157, tlen=871, kl=0.242, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  91%|█████████▏| 117/128 [06:12<00:27,  2.51s/it, pg=-0.234, rm=0.875, ret=0.218, glen=150, tlen=977, kl=0.25, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  92%|█████████▏| 118/128 [06:12<00:23,  2.38s/it, pg=-0.234, rm=0.875, ret=0.218, glen=150, tlen=977, kl=0.25, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  92%|█████████▏| 118/128 [06:13<00:23,  2.38s/it, pg=0.0897, rm=0.875, ret=-0.109, glen=162, tlen=848, kl=0.263, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  93%|█████████▎| 119/128 [06:13<00:20,  2.22s/it, pg=0.0897, rm=0.875, ret=-0.109, glen=162, tlen=848, kl=0.263, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  93%|█████████▎| 119/128 [06:24<00:20,  2.22s/it, pg=-0.0813, rm=0.75, ret=0.0624, glen=160, tlen=863, kl=0.273, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  94%|█████████▍| 120/128 [06:24<00:37,  4.65s/it, pg=-0.0813, rm=0.75, ret=0.0624, glen=160, tlen=863, kl=0.273, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  94%|█████████▍| 120/128 [06:26<00:37,  4.65s/it, pg=0.0699, rm=0.938, ret=-0.0899, glen=155, tlen=914, kl=0.277, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  95%|█████████▍| 121/128 [06:26<00:26,  3.83s/it, pg=0.0699, rm=0.938, ret=-0.0899, glen=155, tlen=914, kl=0.277, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  95%|█████████▍| 121/128 [06:28<00:26,  3.83s/it, pg=0.115, rm=1, ret=-0.127, glen=145, tlen=968, kl=0.239, act_lr=9.59e-7]      
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  95%|█████████▌| 122/128 [06:28<00:19,  3.30s/it, pg=0.115, rm=1, ret=-0.127, glen=145, tlen=968, kl=0.239, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  95%|█████████▌| 122/128 [06:30<00:19,  3.30s/it, pg=0.0576, rm=1.06, ret=-0.0753, glen=159, tlen=957, kl=0.239, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  96%|█████████▌| 123/128 [06:30<00:14,  2.93s/it, pg=0.0576, rm=1.06, ret=-0.0753, glen=159, tlen=957, kl=0.239, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  96%|█████████▌| 123/128 [06:32<00:14,  2.93s/it, pg=0.0921, rm=0.812, ret=-0.109, glen=144, tlen=976, kl=0.262, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  97%|█████████▋| 124/128 [06:32<00:10,  2.68s/it, pg=0.0921, rm=0.812, ret=-0.109, glen=144, tlen=976, kl=0.262, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  97%|█████████▋| 124/128 [06:34<00:10,  2.68s/it, pg=-0.205, rm=1.06, ret=0.188, glen=149, tlen=1029.75, kl=0.256, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  98%|█████████▊| 125/128 [06:34<00:07,  2.54s/it, pg=-0.205, rm=1.06, ret=0.188, glen=149, tlen=1029.75, kl=0.256, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  98%|█████████▊| 125/128 [06:36<00:07,  2.54s/it, pg=-0.0336, rm=1, ret=0.0145, glen=137, tlen=1.01e+3, kl=0.273, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  98%|█████████▊| 126/128 [06:36<00:04,  2.45s/it, pg=-0.0336, rm=1, ret=0.0145, glen=137, tlen=1.01e+3, kl=0.273, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  98%|█████████▊| 126/128 [06:38<00:04,  2.45s/it, pg=0.0871, rm=0.75, ret=-0.104, glen=151, tlen=1.04e+3, kl=0.248, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  99%|█████████▉| 127/128 [06:38<00:02,  2.38s/it, pg=0.0871, rm=0.75, ret=-0.104, glen=151, tlen=1.04e+3, kl=0.248, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  99%|█████████▉| 127/128 [06:48<00:02,  2.38s/it, pg=-0.207, rm=1.06, ret=0.188, glen=152, tlen=983, kl=0.251, act_lr=9.59e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]: 100%|██████████| 128/128 [06:48<00:00,  4.62s/it, pg=-0.207, rm=1.06, ret=0.188, glen=152, tlen=983, kl=0.251, act_lr=9.59e-7]
+Train epoch [1/1]: 100%|██████████| 128/128 [06:48<00:00,  3.19s/it, pg=-0.207, rm=1.06, ret=0.188, glen=152, tlen=983, kl=0.251, act_lr=9.59e-7]
+(LLMRayActor pid=425166) INFO 05-28 00:13:46 [executor_base.py:219] It took 1.312600 seconds to wake up.
+(LLMRayActor pid=425166) update weight: visual.patch_embed.proj.weight, dtype: torch.bfloat16, shape: torch.Size([1280, 3, 2, 14, 14])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.norm1.weight, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.norm2.weight, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.attn.qkv.weight, dtype: torch.bfloat16, shape: torch.Size([3840, 1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.attn.qkv.bias, dtype: torch.bfloat16, shape: torch.Size([3840])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.attn.proj.weight, dtype: torch.bfloat16, shape: torch.Size([1280, 1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.attn.proj.bias, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: visual.blocks.0.mlp.gate_proj.weight, dtype: torch.bfloat16, shape: torch.Size([3420, 1280])
+(LLMRayActor pid=425166) update weight: visual.merger.ln_q.weight, dtype: torch.bfloat16, shape: torch.Size([1280])
+(LLMRayActor pid=425166) update weight: model.embed_tokens.weight, dtype: torch.bfloat16, shape: torch.Size([152064, 3584])
+(LLMRayActor pid=425165) INFO 05-28 00:13:48 [executor_base.py:219] It took 3.195586 seconds to wake up. [repeated 7x across cluster]
+(LLMRayActor pid=425165) update weight: visual.patch_embed.proj.weight, dtype: torch.bfloat16, shape: torch.Size([1280, 3, 2, 14, 14]) [repeated 7x across cluster]
+(LLMRayActor pid=425166) update weight: model.layers.2.mlp.up_proj.weight, dtype: torch.bfloat16, shape: torch.Size([18944, 3584]) [repeated 3340x across cluster]
+(LLMRayActor pid=425165) update weight: visual.merger.ln_q.weight, dtype: torch.bfloat16, shape: torch.Size([1280]) [repeated 7x across cluster]
+(LLMRayActor pid=425165) update weight: model.embed_tokens.weight, dtype: torch.bfloat16, shape: torch.Size([152064, 3584]) [repeated 7x across cluster]
+(LLMRayActor pid=425166) update weight: model.layers.12.mlp.up_proj.weight, dtype: torch.bfloat16, shape: torch.Size([18944, 3584]) [repeated 941x across cluster]
+(LLMRayActor pid=425166) update weight: model.layers.22.mlp.gate_proj.weight, dtype: torch.bfloat16, shape: torch.Size([18944, 3584]) [repeated 942x across cluster]
+(LLMRayActor pid=425166) update weight: model.norm.weight, dtype: torch.bfloat16, shape: torch.Size([3584])
+(LLMRayActor pid=425166) update weight: lm_head.weight, dtype: torch.bfloat16, shape: torch.Size([152064, 3584])
+(LLMRayActor pid=425166) INFO 05-28 00:14:09 [worker.py:133] Sleep mode freed 38.53 GiB memory, 18.76 GiB memory is still in use.
+(LLMRayActor pid=425166) INFO 05-28 00:14:09 [executor_base.py:208] It took 1.409277 seconds to fall asleep.
+(LLMRayActor pid=425165) update weight: model.layers.27.post_attention_layernorm.weight, dtype: torch.bfloat16, shape: torch.Size([3584]) [repeated 562x across cluster]
+(ActorModelRayActor pid=430510) 
+Episode [1/2]:   0%|          | 0/127 [12:55<?, ?it/s, policy_loss=0.0208, actor_lr=9.6e-7, kl=0.252, reward=0.93, response_length=154, total_length=911, acc_rewards=0.43, format_rewards=0.5, return=-0.0389]
+Episode [1/2]:   1%|          | 1/127 [12:55<27:08:37, 775.53s/it, policy_loss=0.0208, actor_lr=9.6e-7, kl=0.252, reward=0.93, response_length=154, total_length=911, acc_rewards=0.43, format_rewards=0.5, return=-0.0389]
+(LLMRayActor pid=425166) INFO 05-28 00:14:11 [executor_base.py:219] It took 1.523881 seconds to wake up.
+(LLMRayActor pid=425165) update weight: model.norm.weight, dtype: torch.bfloat16, shape: torch.Size([3584]) [repeated 7x across cluster]
+(LLMRayActor pid=425165) update weight: lm_head.weight, dtype: torch.bfloat16, shape: torch.Size([152064, 3584]) [repeated 7x across cluster]
+(LLMRayActor pid=425161) 
+Processed prompts:   0%|          | 0/256 [00:00<?, ?it/s, est. speed input: 0.00 toks/s, output: 0.00 toks/s]
+(LLMRayActor pid=425164) 
+Processed prompts:   0%|          | 0/256 [00:00<?, ?it/s, est. speed input: 0.00 toks/s, output: 0.00 toks/s] [repeated 7x across cluster]
+(LLMRayActor pid=425161) 
+Processed prompts:   0%|          | 1/256 [00:19<1:22:59, 19.53s/it, est. speed input: 79.22 toks/s, output: 5.27 toks/s]
+Processed prompts:   1%|          | 2/256 [00:19<34:21,  8.12s/it, est. speed input: 98.99 toks/s, output: 10.63 toks/s] 
+(LLMRayActor pid=425161) 
+Processed prompts:   1%|          | 3/256 [00:19<18:51,  4.47s/it, est. speed input: 118.47 toks/s, output: 16.07 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:   2%|▏         | 5/256 [00:20<08:22,  2.00s/it, est. speed input: 221.02 toks/s, output: 27.14 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:   4%|▍         | 11/256 [00:20<02:29,  1.64it/s, est. speed input: 359.20 toks/s, output: 61.32 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:   6%|▌         | 15/256 [00:20<01:31,  2.62it/s, est. speed input: 513.39 toks/s, output: 84.58 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:   8%|▊         | 20/256 [00:20<00:54,  4.30it/s, est. speed input: 642.84 toks/s, output: 114.52 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  10%|▉         | 25/256 [00:20<00:35,  6.45it/s, est. speed input: 745.72 toks/s, output: 144.66 toks/s]
+Processed prompts:  12%|█▎        | 32/256 [00:20<00:21, 10.41it/s, est. speed input: 992.19 toks/s, output: 187.77 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  16%|█▌        | 40/256 [00:20<00:13, 15.95it/s, est. speed input: 1267.54 toks/s, output: 237.73 toks/s]
+Processed prompts:  18%|█▊        | 45/256 [00:21<00:11, 19.17it/s, est. speed input: 1365.56 toks/s, output: 269.01 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  21%|██        | 54/256 [00:21<00:07, 27.63it/s, est. speed input: 1558.53 toks/s, output: 326.92 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  27%|██▋       | 69/256 [00:21<00:04, 45.30it/s, est. speed input: 2036.93 toks/s, output: 425.75 toks/s]
+Processed prompts:  32%|███▏      | 82/256 [00:21<00:02, 59.44it/s, est. speed input: 2304.93 toks/s, output: 511.93 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  36%|███▌      | 92/256 [00:21<00:02, 66.94it/s, est. speed input: 2522.61 toks/s, output: 578.73 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  43%|████▎     | 109/256 [00:21<00:01, 83.78it/s, est. speed input: 2961.55 toks/s, output: 694.71 toks/s]
+Processed prompts:  47%|████▋     | 121/256 [00:21<00:01, 86.87it/s, est. speed input: 3216.26 toks/s, output: 776.96 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  52%|█████▏    | 132/256 [00:21<00:01, 89.23it/s, est. speed input: 3529.64 toks/s, output: 853.26 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  59%|█████▊    | 150/256 [00:21<00:00, 109.26it/s, est. speed input: 3962.29 toks/s, output: 983.58 toks/s]
+Processed prompts:  65%|██████▍   | 166/256 [00:22<00:00, 109.41it/s, est. speed input: 4318.90 toks/s, output: 1099.12 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  72%|███████▏  | 184/256 [00:22<00:00, 120.88it/s, est. speed input: 4695.77 toks/s, output: 1234.89 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  78%|███████▊  | 200/256 [00:22<00:00, 129.28it/s, est. speed input: 5127.98 toks/s, output: 1357.86 toks/s]
+Processed prompts:  85%|████████▍ | 217/256 [00:22<00:00, 135.69it/s, est. speed input: 5478.87 toks/s, output: 1491.54 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  91%|█████████ | 232/256 [00:22<00:00, 139.34it/s, est. speed input: 5811.80 toks/s, output: 1612.11 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts:  96%|█████████▋| 247/256 [00:22<00:00, 119.01it/s, est. speed input: 6071.44 toks/s, output: 1731.74 toks/s]
+(LLMRayActor pid=425161) 
+Processed prompts: 100%|██████████| 256/256 [00:25<00:00, 10.15it/s, est. speed input: 5635.83 toks/s, output: 1645.10 toks/s] 
+(LLMRayActor pid=425163) 
+Processed prompts:   0%|          | 1/256 [00:25<1:46:53, 25.15s/it, est. speed input: 61.35 toks/s, output: 3.38 toks/s]
+(LLMRayActor pid=425160) 
+Processed prompts:   0%|          | 1/256 [00:25<1:49:44, 25.82s/it, est. speed input: 59.95 toks/s, output: 3.41 toks/s]
+Processed prompts:   1%|          | 3/256 [00:25<28:26,  6.74s/it, est. speed input: 178.92 toks/s, output: 10.33 toks/s]
+(LLMRayActor pid=425163) 
+Processed prompts:  96%|█████████▌| 245/256 [00:28<00:00, 102.62it/s, est. speed input: 6344.77 toks/s, output: 1269.65 toks/s]
+(LLMRayActor pid=425163) 
+Processed prompts: 100%|██████████| 256/256 [00:29<00:00, 47.62it/s, est. speed input: 6466.57 toks/s, output: 1324.54 toks/s] 
+Processed prompts: 100%|██████████| 256/256 [00:29<00:00,  8.69it/s, est. speed input: 6466.57 toks/s, output: 1324.54 toks/s]
+(LLMRayActor pid=425162) 
+Processed prompts:   7%|▋         | 18/256 [00:28<00:59,  3.97it/s, est. speed input: 746.60 toks/s, output: 67.98 toks/s] [repeated 48x across cluster]
+(LLMRayActor pid=425162) 
+Processed prompts:  18%|█▊        | 46/256 [00:28<00:09, 22.01it/s, est. speed input: 1979.50 toks/s, output: 188.46 toks/s]
+Processed prompts:  21%|██        | 54/256 [00:29<00:06, 29.61it/s, est. speed input: 2339.24 toks/s, output: 224.72 toks/s] [repeated 31x across cluster]
+(LLMRayActor pid=425162) 
+Processed prompts:  98%|█████████▊| 250/256 [00:30<00:00, 82.50it/s, est. speed input: 6528.19 toks/s, output: 1275.52 toks/s]  [repeated 11x across cluster]
+(LLMRayActor pid=425164) 
+Processed prompts:  20%|██        | 52/256 [00:29<00:13, 14.78it/s, est. speed input: 1889.84 toks/s, output: 202.88 toks/s] [repeated 29x across cluster]
+(LLMRayActor pid=425164) 
+Processed prompts:  36%|███▌      | 92/256 [00:29<00:03, 48.30it/s, est. speed input: 2931.29 toks/s, output: 377.94 toks/s]
+Processed prompts:  40%|███▉      | 102/256 [00:30<00:02, 57.19it/s, est. speed input: 3212.82 toks/s, output: 423.50 toks/s] [repeated 17x across cluster]
+(LLMRayActor pid=425162) 
+Processed prompts: 100%|█████████▉| 255/256 [00:45<00:00, 82.50it/s, est. speed input: 6526.09 toks/s, output: 1296.31 toks/s] [repeated 8x across cluster]
+(LLMRayActor pid=425164) 
+Processed prompts:  88%|████████▊ | 224/256 [00:31<00:00, 101.00it/s, est. speed input: 5999.71 toks/s, output: 1048.46 toks/s] [repeated 5x across cluster]
+(LLMRayActor pid=425164) 
+Processed prompts:  79%|███████▊  | 201/256 [00:31<00:00, 99.90it/s, est. speed input: 5508.21 toks/s, output: 918.56 toks/s] 
+Processed prompts:  83%|████████▎ | 212/256 [00:31<00:00, 98.07it/s, est. speed input: 5693.72 toks/s, output: 979.57 toks/s] [repeated 3x across cluster]
+(LLMRayActor pid=425162) 
+Processed prompts: 100%|██████████| 256/256 [01:18<00:00,  1.26s/it, est. speed input: 2597.64 toks/s, output: 567.02 toks/s] 
+Processed prompts: 100%|██████████| 256/256 [01:18<00:00,  3.24it/s, est. speed input: 2597.64 toks/s, output: 567.02 toks/s]
+(ActorModelRayActor pid=431329) ele.get("min_pixels" 3136
+(ActorModelRayActor pid=431329) ele.get("max_pixels" 1254400
+(LLMRayActor pid=425165) INFO 05-28 00:14:09 [worker.py:133] Sleep mode freed 38.53 GiB memory, 19.00 GiB memory is still in use. [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-28 00:14:09 [executor_base.py:208] It took 1.616499 seconds to fall asleep. [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-28 00:14:13 [executor_base.py:219] It took 3.164235 seconds to wake up. [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) ele.get("min_pixels" 3136 [repeated 845x across cluster]
+(ActorModelRayActor pid=430510) ele.get("max_pixels" 1254400 [repeated 845x across cluster]
+(ActorModelRayActor pid=430510)  1254400
+(ActorModelRayActor pid=430510) ele.get("min_pixels" 3136 [repeated 855x across cluster]
+(ActorModelRayActor pid=430510) ele.get("max_pixels" 1254400 [repeated 855x across cluster]
+(LLMRayActor pid=425166) INFO 05-28 00:16:38 [worker.py:133] Sleep mode freed 38.53 GiB memory, 21.45 GiB memory is still in use.
+(LLMRayActor pid=425166) INFO 05-28 00:16:38 [executor_base.py:208] It took 1.395119 seconds to fall asleep.
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   0%|          | 0/128 [00:00<?, ?it/s]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   1%|          | 1/128 [00:01<03:33,  1.68s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   2%|▏         | 2/128 [00:03<03:14,  1.55s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   2%|▏         | 3/128 [00:04<03:16,  1.57s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   3%|▎         | 4/128 [00:06<03:12,  1.55s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   4%|▍         | 5/128 [00:07<03:04,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   5%|▍         | 6/128 [00:09<02:58,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   5%|▌         | 7/128 [00:10<02:55,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   6%|▋         | 8/128 [00:11<02:51,  1.43s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   7%|▋         | 9/128 [00:13<02:47,  1.41s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   8%|▊         | 10/128 [00:14<02:43,  1.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   9%|▊         | 11/128 [00:16<02:44,  1.41s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:   9%|▉         | 12/128 [00:17<02:41,  1.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  10%|█         | 13/128 [00:18<02:39,  1.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  11%|█         | 14/128 [00:20<02:40,  1.40s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  12%|█▏        | 15/128 [00:21<02:38,  1.40s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  12%|█▎        | 16/128 [00:22<02:36,  1.40s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  13%|█▎        | 17/128 [00:24<02:32,  1.37s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  14%|█▍        | 18/128 [00:25<02:31,  1.38s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  15%|█▍        | 19/128 [00:26<02:26,  1.34s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  16%|█▌        | 20/128 [00:28<02:24,  1.33s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  16%|█▋        | 21/128 [00:30<02:46,  1.56s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  17%|█▋        | 22/128 [00:31<02:40,  1.51s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  18%|█▊        | 23/128 [00:33<02:34,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  19%|█▉        | 24/128 [00:34<02:37,  1.51s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  20%|█▉        | 25/128 [00:36<02:33,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  20%|██        | 26/128 [00:37<02:28,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  21%|██        | 27/128 [00:38<02:26,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  22%|██▏       | 28/128 [00:40<02:24,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  23%|██▎       | 29/128 [00:41<02:20,  1.42s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  23%|██▎       | 30/128 [00:43<02:19,  1.42s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  24%|██▍       | 31/128 [00:44<02:17,  1.41s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  25%|██▌       | 32/128 [00:46<02:18,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  26%|██▌       | 33/128 [00:47<02:11,  1.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  27%|██▋       | 34/128 [00:48<02:11,  1.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  27%|██▋       | 35/128 [00:50<02:09,  1.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  28%|██▊       | 36/128 [00:51<02:06,  1.37s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  29%|██▉       | 37/128 [00:52<02:02,  1.35s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  30%|██▉       | 38/128 [00:54<02:03,  1.37s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  30%|███       | 39/128 [00:55<02:01,  1.37s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  31%|███▏      | 40/128 [00:56<01:58,  1.34s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  32%|███▏      | 41/128 [00:58<02:00,  1.38s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  33%|███▎      | 42/128 [00:59<02:01,  1.41s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  34%|███▎      | 43/128 [01:01<02:03,  1.45s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  34%|███▍      | 44/128 [01:02<02:05,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  35%|███▌      | 45/128 [01:04<02:04,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  36%|███▌      | 46/128 [01:06<02:06,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  37%|███▋      | 47/128 [01:07<02:03,  1.53s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  38%|███▊      | 48/128 [01:09<02:04,  1.56s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  38%|███▊      | 49/128 [01:10<02:03,  1.57s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  39%|███▉      | 50/128 [01:12<02:01,  1.56s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  40%|███▉      | 51/128 [01:13<01:58,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  41%|████      | 52/128 [01:15<01:59,  1.57s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  41%|████▏     | 53/128 [01:16<01:55,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  42%|████▏     | 54/128 [01:18<01:54,  1.55s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  43%|████▎     | 55/128 [01:20<01:52,  1.55s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  44%|████▍     | 56/128 [01:21<01:50,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  45%|████▍     | 57/128 [01:22<01:38,  1.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  45%|████▌     | 58/128 [01:23<01:27,  1.25s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  46%|████▌     | 59/128 [01:24<01:20,  1.17s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  47%|████▋     | 60/128 [01:25<01:16,  1.13s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  48%|████▊     | 61/128 [01:26<01:13,  1.09s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  48%|████▊     | 62/128 [01:27<01:11,  1.08s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  49%|████▉     | 63/128 [01:28<01:08,  1.05s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  50%|█████     | 64/128 [01:29<01:05,  1.02s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  51%|█████     | 65/128 [01:31<01:12,  1.16s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  52%|█████▏    | 66/128 [01:32<01:15,  1.22s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  52%|█████▏    | 67/128 [01:33<01:20,  1.32s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  53%|█████▎    | 68/128 [01:35<01:21,  1.35s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  54%|█████▍    | 69/128 [01:36<01:20,  1.37s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  55%|█████▍    | 70/128 [01:38<01:20,  1.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  55%|█████▌    | 71/128 [01:39<01:20,  1.41s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  56%|█████▋    | 72/128 [01:41<01:20,  1.44s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  57%|█████▋    | 73/128 [01:42<01:21,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  58%|█████▊    | 74/128 [01:44<01:20,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  59%|█████▊    | 75/128 [01:45<01:18,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  59%|███���█▉    | 76/128 [01:47<01:16,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  60%|██████    | 77/128 [01:48<01:16,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  61%|██████    | 78/128 [01:50<01:14,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  62%|██████▏   | 79/128 [01:51<01:13,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  62%|██████▎   | 80/128 [01:53<01:11,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  63%|██████▎   | 81/128 [01:54<01:09,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  64%|██████▍   | 82/128 [01:56<01:07,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  65%|██████▍   | 83/128 [01:57<01:08,  1.53s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  66%|██████▌   | 84/128 [01:59<01:06,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  66%|██████▋   | 85/128 [02:00<01:05,  1.52s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  67%|██████▋   | 86/128 [02:02<01:02,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  68%|██████▊   | 87/128 [02:03<01:01,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  69%|██████▉   | 88/128 [02:05<00:58,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  70%|██████▉   | 89/128 [02:06<00:58,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  70%|███████   | 90/128 [02:08<00:56,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  71%|███████   | 91/128 [02:09<00:55,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  72%|███████▏  | 92/128 [02:11<00:53,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  73%|███████▎  | 93/128 [02:12<00:51,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  73%|███████▎  | 94/128 [02:14<00:50,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  74%|███████▍  | 95/128 [02:15<00:49,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  75%|███████▌  | 96/128 [02:17<00:48,  1.51s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  76%|███████▌  | 97/128 [02:18<00:45,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  77%|███████▋  | 98/128 [02:20<00:44,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  77%|███████▋  | 99/128 [02:21<00:42,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  78%|███████▊  | 100/128 [02:22<00:40,  1.46s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  79%|███████▉  | 101/128 [02:24<00:39,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  80%|███████▉  | 102/128 [02:25<00:38,  1.48s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  80%|████████  | 103/128 [02:27<00:37,  1.49s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  81%|████████▏ | 104/128 [02:28<00:35,  1.47s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  82%|████████▏ | 105/128 [02:29<00:30,  1.34s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  83%|████████▎ | 106/128 [02:30<00:27,  1.26s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  84%|████████▎ | 107/128 [02:31<00:24,  1.18s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  84%|████████▍ | 108/128 [02:32<00:22,  1.12s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  85%|████████▌ | 109/128 [02:33<00:20,  1.08s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  86%|████████▌ | 110/128 [02:34<00:19,  1.06s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  87%|████████▋ | 111/128 [02:35<00:17,  1.05s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  88%|████████▊ | 112/128 [02:37<00:16,  1.06s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  88%|████████▊ | 113/128 [02:38<00:17,  1.15s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  89%|████████▉ | 114/128 [02:39<00:17,  1.24s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  90%|████████▉ | 115/128 [02:41<00:16,  1.29s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  91%|█████████ | 116/128 [02:42<00:16,  1.36s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  91%|█████████▏| 117/128 [02:44<00:15,  1.39s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  92%|█████████▏| 118/128 [02:45<00:14,  1.40s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  93%|█████████▎| 119/128 [02:47<00:12,  1.41s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  94%|█████████▍| 120/128 [02:48<00:11,  1.42s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  95%|█████████▍| 121/128 [02:50<00:10,  1.46s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  95%|█████████▌| 122/128 [02:51<00:08,  1.50s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  96%|█████████▌| 123/128 [02:53<00:07,  1.51s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  97%|█████████▋| 124/128 [02:54<00:06,  1.54s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  98%|█████████▊| 125/128 [02:56<00:04,  1.57s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  98%|█████████▊| 126/128 [02:58<00:03,  1.57s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience:  99%|█████████▉| 127/128 [02:59<00:01,  1.56s/it]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+make_experience: 100%|██████████| 128/128 [03:01<00:00,  1.57s/it]
+make_experience: 100%|██████████| 128/128 [03:01<00:00,  1.42s/it]
+(ActorModelRayActor pid=430510) ['system\nA conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. The reasoning process and answer are enclosed within <think> </think> and <answer> </answer> tags, respectively, i.e., <think> reasoning process here </think><answer> answer here </answer>\nuser\nHow visually aligned is this with the prompt: "half black character, illustration, portrait"?\nPlease evaluate how well the image matches each element of provided prompt.\n\n And answer with the final alignment rating.\nRate it from 0 to 5 (float, 2 decimals). A rating of 0 represents very poor alignment level, while 5 represents excellent alignment level.\'\nassistant\n<think> The image depicts a character with dark skin, which aligns with the element of "half black character." The style is an illustration, and the composition is a portrait. The elements of the prompt are well-represented, but the term "half black character" might be slightly misleading as the image does not provide any indication of the character\'s ethnicity beyond the skin tone. The focus is on the character\'s face and upper body, making it a portrait, but there is no indication of the character being "half" anything. The image is visually engaging and styled appropriately for an illustration.</think>\n<answer>3.50</answer>']
+(ActorModelRayActor pid=431330) ele.get("min_pixels" 3136 [repeated 347x across cluster]
+(ActorModelRayActor pid=431330) ele.get("max_pixels" 1254400 [repeated 347x across cluster]
+(LLMRayActor pid=425165) INFO 05-28 00:16:38 [worker.py:133] Sleep mode freed 38.22 GiB memory, 22.00 GiB memory is still in use. [repeated 7x across cluster]
+(LLMRayActor pid=425165) INFO 05-28 00:16:38 [executor_base.py:208] It took 1.715809 seconds to fall asleep. [repeated 7x across cluster]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   0%|          | 0/128 [00:00<?, ?it/s]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   0%|          | 0/128 [00:02<?, ?it/s, pg=-0.0213, rm=0.812, ret=0.000132, glen=159, tlen=875, kl=0.271, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   1%|          | 1/128 [00:02<05:31,  2.61s/it, pg=-0.0213, rm=0.812, ret=0.000132, glen=159, tlen=875, kl=0.271, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   1%|          | 1/128 [00:04<05:31,  2.61s/it, pg=-0.111, rm=0.938, ret=0.0929, glen=151, tlen=765, kl=0.258, act_lr=9.59e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   2%|▏         | 2/128 [00:04<04:57,  2.36s/it, pg=-0.111, rm=0.938, ret=0.0929, glen=151, tlen=765, kl=0.258, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   2%|▏         | 2/128 [00:07<04:57,  2.36s/it, pg=0.0834, rm=0.875, ret=-0.102, glen=154, tlen=1.06e+3, kl=0.262, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   2%|▏         | 3/128 [00:07<04:46,  2.29s/it, pg=0.0834, rm=0.875, ret=-0.102, glen=154, tlen=1.06e+3, kl=0.262, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   2%|▏         | 3/128 [00:09<04:46,  2.29s/it, pg=0.127, rm=1, ret=-0.148, glen=173, tlen=792, kl=0.254, act_lr=9.59e-7]         
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   3%|▎         | 4/128 [00:09<04:34,  2.21s/it, pg=0.127, rm=1, ret=-0.148, glen=173, tlen=792, kl=0.254, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   3%|▎         | 4/128 [00:11<04:34,  2.21s/it, pg=-0.0958, rm=1.06, ret=0.077, glen=156, tlen=887, kl=0.263, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   4%|▍         | 5/128 [00:11<04:19,  2.11s/it, pg=-0.0958, rm=1.06, ret=0.077, glen=156, tlen=887, kl=0.263, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   4%|▍         | 5/128 [00:13<04:19,  2.11s/it, pg=-0.115, rm=1, ret=0.0959, glen=160, tlen=791, kl=0.264, act_lr=9.59e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   5%|▍         | 6/128 [00:13<04:18,  2.12s/it, pg=-0.115, rm=1, ret=0.0959, glen=160, tlen=791, kl=0.264, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   5%|▍         | 6/128 [00:15<04:18,  2.12s/it, pg=0.0212, rm=1.25, ret=-0.0416, glen=146, tlen=965, kl=0.285, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   5%|▌         | 7/128 [00:15<04:12,  2.09s/it, pg=0.0212, rm=1.25, ret=-0.0416, glen=146, tlen=965, kl=0.285, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   5%|▌         | 7/128 [00:26<04:12,  2.09s/it, pg=0.0701, rm=0.938, ret=-0.0904, glen=158, tlen=928, kl=0.272, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   6%|▋         | 8/128 [00:26<09:59,  4.99s/it, pg=0.0701, rm=0.938, ret=-0.0904, glen=158, tlen=928, kl=0.272, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   6%|▋         | 8/128 [00:28<09:59,  4.99s/it, pg=-0.0375, rm=0.938, ret=0.0187, glen=156, tlen=1.06e+3, kl=0.259, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   7%|▋         | 9/128 [00:28<08:15,  4.17s/it, pg=-0.0375, rm=0.938, ret=0.0187, glen=156, tlen=1.06e+3, kl=0.259, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   7%|▋         | 9/128 [00:30<08:15,  4.17s/it, pg=0.0225, rm=1, ret=-0.0426, glen=154, tlen=929, kl=0.277, act_lr=9.59e-7]        
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   8%|▊         | 10/128 [00:30<06:55,  3.52s/it, pg=0.0225, rm=1, ret=-0.0426, glen=154, tlen=929, kl=0.277, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   8%|▊         | 10/128 [00:33<06:55,  3.52s/it, pg=0.158, rm=0.688, ret=-0.179, glen=174, tlen=842, kl=0.254, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   9%|▊         | 11/128 [00:33<06:17,  3.23s/it, pg=0.158, rm=0.688, ret=-0.179, glen=174, tlen=842, kl=0.254, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   9%|▊         | 11/128 [00:35<06:17,  3.23s/it, pg=0.0997, rm=1.19, ret=-0.118, glen=159, tlen=853, kl=0.243, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   9%|▉         | 12/128 [00:35<05:28,  2.83s/it, pg=0.0997, rm=1.19, ret=-0.118, glen=159, tlen=853, kl=0.243, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:   9%|▉         | 12/128 [00:37<05:28,  2.83s/it, pg=-0.0583, rm=0.938, ret=0.0395, glen=151, tlen=1.13e+3, kl=0.241, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  10%|█         | 13/128 [00:37<05:13,  2.73s/it, pg=-0.0583, rm=0.938, ret=0.0395, glen=151, tlen=1.13e+3, kl=0.241, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  10%|█         | 13/128 [00:39<05:13,  2.73s/it, pg=-0.0845, rm=0.812, ret=0.067, glen=161, tlen=998, kl=0.244, act_lr=9.59e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  11%|█         | 14/128 [00:39<04:49,  2.54s/it, pg=-0.0845, rm=0.812, ret=0.067, glen=161, tlen=998, kl=0.244, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  11%|█         | 14/128 [00:41<04:49,  2.54s/it, pg=0.00246, rm=1.12, ret=-0.0213, glen=157, tlen=898, kl=0.259, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  12%|█▏        | 15/128 [00:41<04:25,  2.35s/it, pg=0.00246, rm=1.12, ret=-0.0213, glen=157, tlen=898, kl=0.259, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  12%|█▏        | 15/128 [00:52<04:25,  2.35s/it, pg=0.0485, rm=1.06, ret=-0.0646, glen=149, tlen=842, kl=0.247, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  12%|█▎        | 16/128 [00:52<08:57,  4.80s/it, pg=0.0485, rm=1.06, ret=-0.0646, glen=149, tlen=842, kl=0.247, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  12%|█▎        | 16/128 [00:54<08:57,  4.80s/it, pg=0.0136, rm=1.12, ret=-0.0311, glen=150, tlen=965, kl=0.236, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  13%|█▎        | 17/128 [00:54<07:24,  4.00s/it, pg=0.0136, rm=1.12, ret=-0.0311, glen=150, tlen=965, kl=0.236, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  13%|█▎        | 17/128 [00:56<07:24,  4.00s/it, pg=0.0176, rm=1, ret=-0.0378, glen=157, tlen=1.07e+3, kl=0.262, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  14%|█▍        | 18/128 [00:56<06:16,  3.42s/it, pg=0.0176, rm=1, ret=-0.0378, glen=157, tlen=1.07e+3, kl=0.262, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  14%|█▍        | 18/128 [00:58<06:16,  3.42s/it, pg=-0.016, rm=0.812, ret=-0.00223, glen=161, tlen=1.01e+3, kl=0.248, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  15%|█▍        | 19/128 [00:58<05:28,  3.01s/it, pg=-0.016, rm=0.812, ret=-0.00223, glen=161, tlen=1.01e+3, kl=0.248, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  15%|█▍        | 19/128 [01:00<05:28,  3.01s/it, pg=-0.074, rm=1.25, ret=0.0539, glen=161, tlen=796, kl=0.279, act_lr=9.59e-7]       
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  16%|█▌        | 20/128 [01:00<04:55,  2.74s/it, pg=-0.074, rm=1.25, ret=0.0539, glen=161, tlen=796, kl=0.279, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  16%|█▌        | 20/128 [01:02<04:55,  2.74s/it, pg=0.019, rm=1, ret=-0.0364, glen=151, tlen=1.08e+3, kl=0.242, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  16%|█▋        | 21/128 [01:02<04:38,  2.60s/it, pg=0.019, rm=1, ret=-0.0364, glen=151, tlen=1.08e+3, kl=0.242, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  16%|█▋        | 21/128 [01:04<04:38,  2.60s/it, pg=0.132, rm=0.938, ret=-0.152, glen=154, tlen=894, kl=0.273, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  17%|█▋        | 22/128 [01:04<04:17,  2.43s/it, pg=0.132, rm=0.938, ret=-0.152, glen=154, tlen=894, kl=0.273, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  17%|█▋        | 22/128 [01:07<04:17,  2.43s/it, pg=-0.163, rm=0.688, ret=0.144, glen=153, tlen=969, kl=0.24, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  18%|█▊        | 23/128 [01:07<04:19,  2.47s/it, pg=-0.163, rm=0.688, ret=0.144, glen=153, tlen=969, kl=0.24, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  18%|█▊        | 23/128 [01:17<04:19,  2.47s/it, pg=0.0241, rm=1, ret=-0.041, glen=175, tlen=927, kl=0.228, act_lr=9.59e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  19%|█▉        | 24/128 [01:17<07:59,  4.61s/it, pg=0.0241, rm=1, ret=-0.041, glen=175, tlen=927, kl=0.228, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  19%|█▉        | 24/128 [01:19<07:59,  4.61s/it, pg=-0.0999, rm=1.25, ret=0.0804, glen=160, tlen=715, kl=0.264, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  20%|█▉        | 25/128 [01:19<06:39,  3.88s/it, pg=-0.0999, rm=1.25, ret=0.0804, glen=160, tlen=715, kl=0.264, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  20%|█▉        | 25/128 [01:21<06:39,  3.88s/it, pg=0.106, rm=1.06, ret=-0.124, glen=157, tlen=724, kl=0.258, act_lr=9.59e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  20%|██        | 26/128 [01:21<05:31,  3.25s/it, pg=0.106, rm=1.06, ret=-0.124, glen=157, tlen=724, kl=0.258, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  20%|██        | 26/128 [01:23<05:31,  3.25s/it, pg=-0.139, rm=1.12, ret=0.118, glen=160, tlen=779, kl=0.27, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  21%|██        | 27/128 [01:23<04:51,  2.89s/it, pg=-0.139, rm=1.12, ret=0.118, glen=160, tlen=779, kl=0.27, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  21%|██        | 27/128 [01:24<04:51,  2.89s/it, pg=-0.0949, rm=1, ret=0.0771, glen=145, tlen=759, kl=0.251, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  22%|██▏       | 28/128 [01:24<04:16,  2.57s/it, pg=-0.0949, rm=1, ret=0.0771, glen=145, tlen=759, kl=0.251, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  22%|██▏       | 28/128 [01:27<04:16,  2.57s/it, pg=-0.0344, rm=1.06, ret=0.0109, glen=163, tlen=795, kl=0.288, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  23%|██▎       | 29/128 [01:27<04:00,  2.43s/it, pg=-0.0344, rm=1.06, ret=0.0109, glen=163, tlen=795, kl=0.288, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  23%|██▎       | 29/128 [01:28<04:00,  2.43s/it, pg=-0.0151, rm=1.12, ret=-0.00424, glen=158, tlen=989, kl=0.259, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  23%|██▎       | 30/128 [01:28<03:42,  2.27s/it, pg=-0.0151, rm=1.12, ret=-0.00424, glen=158, tlen=989, kl=0.259, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  23%|██▎       | 30/128 [01:31<03:42,  2.27s/it, pg=-0.0157, rm=1.19, ret=-0.00288, glen=163, tlen=718, kl=0.254, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  24%|██▍       | 31/128 [01:31<03:35,  2.22s/it, pg=-0.0157, rm=1.19, ret=-0.00288, glen=163, tlen=718, kl=0.254, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  24%|██▍       | 31/128 [01:40<03:35,  2.22s/it, pg=0.0644, rm=1.06, ret=-0.0811, glen=138, tlen=1.07e+3, kl=0.246, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  25%|██▌       | 32/128 [01:40<07:03,  4.41s/it, pg=0.0644, rm=1.06, ret=-0.0811, glen=138, tlen=1.07e+3, kl=0.246, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  25%|██▌       | 32/128 [01:42<07:03,  4.41s/it, pg=-0.0539, rm=1.12, ret=0.0356, glen=147, tlen=836, kl=0.252, act_lr=9.59e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  26%|██▌       | 33/128 [01:42<05:47,  3.66s/it, pg=-0.0539, rm=1.12, ret=0.0356, glen=147, tlen=836, kl=0.252, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  26%|██▌       | 33/128 [01:44<05:47,  3.66s/it, pg=-0.00911, rm=1.12, ret=-0.0079, glen=140, tlen=896, kl=0.272, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  27%|██▋       | 34/128 [01:44<04:58,  3.17s/it, pg=-0.00911, rm=1.12, ret=-0.0079, glen=140, tlen=896, kl=0.272, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  27%|██▋       | 34/128 [01:47<04:58,  3.17s/it, pg=0.124, rm=1, ret=-0.143, glen=142, tlen=782, kl=0.266, act_lr=9.59e-7]       
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  27%|██▋       | 35/128 [01:47<04:38,  2.99s/it, pg=0.124, rm=1, ret=-0.143, glen=142, tlen=782, kl=0.266, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  27%|██▋       | 35/128 [01:49<04:38,  2.99s/it, pg=0.173, rm=0.812, ret=-0.189, glen=141, tlen=972, kl=0.26, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  28%|██▊       | 36/128 [01:49<04:05,  2.67s/it, pg=0.173, rm=0.812, ret=-0.189, glen=141, tlen=972, kl=0.26, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  28%|██▊       | 36/128 [01:51<04:05,  2.67s/it, pg=0.0472, rm=1, ret=-0.064, glen=146, tlen=1179.0, kl=0.25, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  29%|██▉       | 37/128 [01:51<03:46,  2.49s/it, pg=0.0472, rm=1, ret=-0.064, glen=146, tlen=1179.0, kl=0.25, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  29%|██▉       | 37/128 [01:53<03:46,  2.49s/it, pg=-0.0917, rm=1, ret=0.0724, glen=161, tlen=949, kl=0.256, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  30%|██▉       | 38/128 [01:53<03:40,  2.45s/it, pg=-0.0917, rm=1, ret=0.0724, glen=161, tlen=949, kl=0.256, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  30%|██▉       | 38/128 [01:55<03:40,  2.45s/it, pg=0.117, rm=0.625, ret=-0.135, glen=153, tlen=1.06e+3, kl=0.241, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  30%|███       | 39/128 [01:55<03:33,  2.40s/it, pg=0.117, rm=0.625, ret=-0.135, glen=153, tlen=1.06e+3, kl=0.241, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  30%|███       | 39/128 [02:05<03:33,  2.40s/it, pg=-0.0994, rm=1.25, ret=0.0788, glen=161, tlen=716, kl=0.272, act_lr=9.59e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  31%|███▏      | 40/128 [02:05<06:37,  4.52s/it, pg=-0.0994, rm=1.25, ret=0.0788, glen=161, tlen=716, kl=0.272, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  31%|███▏      | 40/128 [02:07<06:37,  4.52s/it, pg=0.181, rm=1.06, ret=-0.201, glen=157, tlen=676, kl=0.272, act_lr=9.59e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  32%|███▏      | 41/128 [02:07<05:22,  3.71s/it, pg=0.181, rm=1.06, ret=-0.201, glen=157, tlen=676, kl=0.272, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  32%|███▏      | 41/128 [02:09<05:22,  3.71s/it, pg=-0.278, rm=0.938, ret=0.258, glen=151, tlen=914, kl=0.268, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  33%|███▎      | 42/128 [02:09<04:38,  3.24s/it, pg=-0.278, rm=0.938, ret=0.258, glen=151, tlen=914, kl=0.268, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  33%|███▎      | 42/128 [02:13<04:38,  3.24s/it, pg=0.175, rm=1.16, ret=-0.195, glen=393, tlen=913, kl=0.099, act_lr=9.59e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  34%|███▎      | 43/128 [02:13<04:55,  3.47s/it, pg=0.175, rm=1.16, ret=-0.195, glen=393, tlen=913, kl=0.099, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  34%|███▎      | 43/128 [02:15<04:55,  3.47s/it, pg=-0.0556, rm=0.938, ret=0.0383, glen=148, tlen=1.13e+3, kl=0.254, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  34%|███▍      | 44/128 [02:15<04:19,  3.09s/it, pg=-0.0556, rm=0.938, ret=0.0383, glen=148, tlen=1.13e+3, kl=0.254, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  34%|███▍      | 44/128 [02:17<04:19,  3.09s/it, pg=-0.0502, rm=0.938, ret=0.0335, glen=155, tlen=887, kl=0.22, act_lr=9.59e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  35%|███▌      | 45/128 [02:17<03:46,  2.73s/it, pg=-0.0502, rm=0.938, ret=0.0335, glen=155, tlen=887, kl=0.22, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  35%|███▌      | 45/128 [02:19<03:46,  2.73s/it, pg=0.0396, rm=0.938, ret=-0.0557, glen=143, tlen=1170.0, kl=0.264, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  36%|███▌      | 46/128 [02:19<03:34,  2.61s/it, pg=0.0396, rm=0.938, ret=-0.0557, glen=143, tlen=1170.0, kl=0.264, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  36%|███▌      | 46/128 [02:22<03:34,  2.61s/it, pg=0.222, rm=0.875, ret=-0.242, glen=158, tlen=928, kl=0.282, act_lr=9.59e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  37%|███▋      | 47/128 [02:22<03:27,  2.56s/it, pg=0.222, rm=0.875, ret=-0.242, glen=158, tlen=928, kl=0.282, act_lr=9.59e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  37%|███▋      | 47/128 [02:31<03:27,  2.56s/it, pg=0.047, rm=1.12, ret=-0.0663, glen=154, tlen=963, kl=0.261, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  38%|███▊      | 48/128 [02:31<06:16,  4.71s/it, pg=0.047, rm=1.12, ret=-0.0663, glen=154, tlen=963, kl=0.261, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  38%|███▊      | 48/128 [02:33<06:16,  4.71s/it, pg=0.0747, rm=0.75, ret=-0.0926, glen=158, tlen=921, kl=0.242, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  38%|███▊      | 49/128 [02:33<05:05,  3.87s/it, pg=0.0747, rm=0.75, ret=-0.0926, glen=158, tlen=921, kl=0.242, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  38%|███▊      | 49/128 [02:35<05:05,  3.87s/it, pg=0.0218, rm=1.06, ret=-0.0417, glen=158, tlen=915, kl=0.264, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  39%|███▉      | 50/128 [02:35<04:21,  3.35s/it, pg=0.0218, rm=1.06, ret=-0.0417, glen=158, tlen=915, kl=0.264, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  39%|███▉      | 50/128 [02:37<04:21,  3.35s/it, pg=0.00878, rm=1.06, ret=-0.0284, glen=158, tlen=956, kl=0.273, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  40%|███▉      | 51/128 [02:37<03:47,  2.96s/it, pg=0.00878, rm=1.06, ret=-0.0284, glen=158, tlen=956, kl=0.273, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  40%|███▉      | 51/128 [02:39<03:47,  2.96s/it, pg=-0.00663, rm=1, ret=-0.0135, glen=158, tlen=844, kl=0.27, act_lr=9.58e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  41%|████      | 52/128 [02:39<03:19,  2.62s/it, pg=-0.00663, rm=1, ret=-0.0135, glen=158, tlen=844, kl=0.27, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  41%|████      | 52/128 [02:41<03:19,  2.62s/it, pg=0.00856, rm=0.938, ret=-0.029, glen=162, tlen=892, kl=0.273, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  41%|████▏     | 53/128 [02:41<03:05,  2.47s/it, pg=0.00856, rm=0.938, ret=-0.029, glen=162, tlen=892, kl=0.273, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  41%|████▏     | 53/128 [02:43<03:05,  2.47s/it, pg=0.00966, rm=1, ret=-0.0267, glen=157, tlen=740, kl=0.238, act_lr=9.58e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  42%|████▏     | 54/128 [02:43<02:49,  2.29s/it, pg=0.00966, rm=1, ret=-0.0267, glen=157, tlen=740, kl=0.238, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  42%|████▏     | 54/128 [02:45<02:49,  2.29s/it, pg=0.137, rm=1.06, ret=-0.155, glen=147, tlen=978, kl=0.252, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  43%|████▎     | 55/128 [02:45<02:37,  2.16s/it, pg=0.137, rm=1.06, ret=-0.155, glen=147, tlen=978, kl=0.252, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  43%|████▎     | 55/128 [02:54<02:37,  2.16s/it, pg=0.0234, rm=1.31, ret=-0.0445, glen=155, tlen=646, kl=0.287, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  44%|████▍     | 56/128 [02:54<05:08,  4.28s/it, pg=0.0234, rm=1.31, ret=-0.0445, glen=155, tlen=646, kl=0.287, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  44%|████▍     | 56/128 [02:57<05:08,  4.28s/it, pg=-0.058, rm=0.938, ret=0.041, glen=144, tlen=1.12e+3, kl=0.244, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  45%|████▍     | 57/128 [02:57<04:27,  3.76s/it, pg=-0.058, rm=0.938, ret=0.041, glen=144, tlen=1.12e+3, kl=0.244, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  45%|████▍     | 57/128 [02:59<04:27,  3.76s/it, pg=0.0262, rm=0.938, ret=-0.0437, glen=147, tlen=1.01e+3, kl=0.255, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  45%|████▌     | 58/128 [02:59<03:43,  3.20s/it, pg=0.0262, rm=0.938, ret=-0.0437, glen=147, tlen=1.01e+3, kl=0.255, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  45%|████▌     | 58/128 [03:01<03:43,  3.20s/it, pg=0.0791, rm=1, ret=-0.0963, glen=157, tlen=933, kl=0.25, act_lr=9.58e-7]         
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  46%|████▌     | 59/128 [03:01<03:19,  2.89s/it, pg=0.0791, rm=1, ret=-0.0963, glen=157, tlen=933, kl=0.25, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  46%|████▌     | 59/128 [03:03<03:19,  2.89s/it, pg=0.0228, rm=1.25, ret=-0.0434, glen=157, tlen=821, kl=0.277, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  47%|████▋     | 60/128 [03:03<02:54,  2.56s/it, pg=0.0228, rm=1.25, ret=-0.0434, glen=157, tlen=821, kl=0.277, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  47%|████▋     | 60/128 [03:05<02:54,  2.56s/it, pg=0.142, rm=1.06, ret=-0.159, glen=144, tlen=885, kl=0.269, act_lr=9.58e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  48%|████▊     | 61/128 [03:05<02:43,  2.44s/it, pg=0.142, rm=1.06, ret=-0.159, glen=144, tlen=885, kl=0.269, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  48%|████▊     | 61/128 [03:07<02:43,  2.44s/it, pg=0.179, rm=1, ret=-0.198, glen=153, tlen=845, kl=0.26, act_lr=9.58e-7]    
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  48%|████▊     | 62/128 [03:07<02:31,  2.30s/it, pg=0.179, rm=1, ret=-0.198, glen=153, tlen=845, kl=0.26, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  48%|████▊     | 62/128 [03:09<02:31,  2.30s/it, pg=0.0986, rm=0.938, ret=-0.118, glen=160, tlen=901, kl=0.256, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  49%|████▉     | 63/128 [03:09<02:24,  2.23s/it, pg=0.0986, rm=0.938, ret=-0.118, glen=160, tlen=901, kl=0.256, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  49%|████▉     | 63/128 [03:19<02:24,  2.23s/it, pg=-0.123, rm=1.06, ret=0.107, glen=149, tlen=1.04e+3, kl=0.247, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  50%|█████     | 64/128 [03:19<04:59,  4.67s/it, pg=-0.123, rm=1.06, ret=0.107, glen=149, tlen=1.04e+3, kl=0.247, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  50%|█████     | 64/128 [03:21<04:59,  4.67s/it, pg=0.181, rm=0.812, ret=-0.192, glen=168, tlen=854, kl=0.225, act_lr=9.58e-7]   
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  51%|█████     | 65/128 [03:21<04:02,  3.85s/it, pg=0.181, rm=0.812, ret=-0.192, glen=168, tlen=854, kl=0.225, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  51%|█████     | 65/128 [03:24<04:02,  3.85s/it, pg=0.0605, rm=0.812, ret=-0.0772, glen=157, tlen=1.13e+3, kl=0.24, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  52%|█████▏    | 66/128 [03:24<03:31,  3.42s/it, pg=0.0605, rm=0.812, ret=-0.0772, glen=157, tlen=1.13e+3, kl=0.24, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  52%|█████▏    | 66/128 [03:26<03:31,  3.42s/it, pg=0.137, rm=0.938, ret=-0.154, glen=148, tlen=1042.5, kl=0.286, act_lr=9.58e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  52%|█████▏    | 67/128 [03:26<03:05,  3.03s/it, pg=0.137, rm=0.938, ret=-0.154, glen=148, tlen=1042.5, kl=0.286, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  52%|█████▏    | 67/128 [03:28<03:05,  3.03s/it, pg=-0.0219, rm=0.812, ret=0.00311, glen=158, tlen=874, kl=0.254, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  53%|█████▎    | 68/128 [03:28<02:47,  2.79s/it, pg=-0.0219, rm=0.812, ret=0.00311, glen=158, tlen=874, kl=0.254, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  53%|█████▎    | 68/128 [03:30<02:47,  2.79s/it, pg=0.0229, rm=1, ret=-0.0429, glen=157, tlen=932, kl=0.273, act_lr=9.58e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  54%|█████▍    | 69/128 [03:30<02:28,  2.51s/it, pg=0.0229, rm=1, ret=-0.0429, glen=157, tlen=932, kl=0.273, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  54%|█████▍    | 69/128 [03:32<02:28,  2.51s/it, pg=-0.00833, rm=1, ret=-0.0104, glen=153, tlen=840, kl=0.257, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  55%|█████▍    | 70/128 [03:32<02:18,  2.39s/it, pg=-0.00833, rm=1, ret=-0.0104, glen=153, tlen=840, kl=0.257, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  55%|█████▍    | 70/128 [03:34<02:18,  2.39s/it, pg=0.0249, rm=1.12, ret=-0.0393, glen=137, tlen=893, kl=0.266, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  55%|█████▌    | 71/128 [03:34<02:13,  2.34s/it, pg=0.0249, rm=1.12, ret=-0.0393, glen=137, tlen=893, kl=0.266, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) [2025-05-28 00:23:29,798] [WARNING] [stage3.py:2139:step] 1 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  55%|█████▌    | 71/128 [03:44<02:13,  2.34s/it, pg=-0.0327, rm=1.31, ret=0.0138, glen=147, tlen=740, kl=0.27, act_lr=9.58e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  56%|█████▋    | 72/128 [03:44<04:17,  4.60s/it, pg=-0.0327, rm=1.31, ret=0.0138, glen=147, tlen=740, kl=0.27, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  56%|█████▋    | 72/128 [03:46<04:17,  4.60s/it, pg=0.179, rm=0.75, ret=-0.195, glen=166, tlen=950, kl=0.247, act_lr=9.58e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  57%|█████▋    | 73/128 [03:46<03:32,  3.86s/it, pg=0.179, rm=0.75, ret=-0.195, glen=166, tlen=950, kl=0.247, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  57%|█████▋    | 73/128 [03:48<03:32,  3.86s/it, pg=0.231, rm=1.06, ret=-0.25, glen=154, tlen=807, kl=0.265, act_lr=9.58e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  58%|█████▊    | 74/128 [03:48<02:58,  3.30s/it, pg=0.231, rm=1.06, ret=-0.25, glen=154, tlen=807, kl=0.265, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  58%|█████▊    | 74/128 [03:50<02:58,  3.30s/it, pg=0.263, rm=0.875, ret=-0.283, glen=165, tlen=828, kl=0.255, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  59%|█████▊    | 75/128 [03:50<02:39,  3.00s/it, pg=0.263, rm=0.875, ret=-0.283, glen=165, tlen=828, kl=0.255, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  59%|█████▊    | 75/128 [03:53<02:39,  3.00s/it, pg=-0.0527, rm=1, ret=0.0296, glen=172, tlen=803, kl=0.271, act_lr=9.58e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  59%|█████▉    | 76/128 [03:53<02:23,  2.75s/it, pg=-0.0527, rm=1, ret=0.0296, glen=172, tlen=803, kl=0.271, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  59%|█████▉    | 76/128 [03:54<02:23,  2.75s/it, pg=0.126, rm=1, ret=-0.145, glen=162, tlen=1.02e+3, kl=0.245, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  60%|██████    | 77/128 [03:54<02:07,  2.49s/it, pg=0.126, rm=1, ret=-0.145, glen=162, tlen=1.02e+3, kl=0.245, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  60%|██████    | 77/128 [03:57<02:07,  2.49s/it, pg=0.0497, rm=0.812, ret=-0.0648, glen=143, tlen=858, kl=0.23, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  61%|██████    | 78/128 [03:57<01:57,  2.36s/it, pg=0.0497, rm=0.812, ret=-0.0648, glen=143, tlen=858, kl=0.23, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  61%|██████    | 78/128 [03:59<01:57,  2.36s/it, pg=0.089, rm=1, ret=-0.108, glen=156, tlen=932, kl=0.266, act_lr=9.58e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  62%|██████▏   | 79/128 [03:59<01:55,  2.35s/it, pg=0.089, rm=1, ret=-0.108, glen=156, tlen=932, kl=0.266, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  62%|██████▏   | 79/128 [04:08<01:55,  2.35s/it, pg=-0.0308, rm=1.25, ret=0.0116, glen=157, tlen=768, kl=0.266, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  62%|██████▎   | 80/128 [04:08<03:36,  4.50s/it, pg=-0.0308, rm=1.25, ret=0.0116, glen=157, tlen=768, kl=0.266, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  62%|██████▎   | 80/128 [04:10<03:36,  4.50s/it, pg=0.0221, rm=1.06, ret=-0.0426, glen=167, tlen=924, kl=0.256, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  63%|██████▎   | 81/128 [04:10<02:57,  3.77s/it, pg=0.0221, rm=1.06, ret=-0.0426, glen=167, tlen=924, kl=0.256, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  63%|██████▎   | 81/128 [04:12<02:57,  3.77s/it, pg=0.0191, rm=1.06, ret=-0.0363, glen=148, tlen=906, kl=0.244, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  64%|██████▍   | 82/128 [04:12<02:29,  3.25s/it, pg=0.0191, rm=1.06, ret=-0.0363, glen=148, tlen=906, kl=0.244, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  64%|██████▍   | 82/128 [04:14<02:29,  3.25s/it, pg=-0.0766, rm=1.06, ret=0.0602, glen=161, tlen=951, kl=0.217, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  65%|██████▍   | 83/128 [04:14<02:07,  2.83s/it, pg=-0.0766, rm=1.06, ret=0.0602, glen=161, tlen=951, kl=0.217, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  65%|██████▍   | 83/128 [04:16<02:07,  2.83s/it, pg=0.0968, rm=1.06, ret=-0.118, glen=168, tlen=874, kl=0.269, act_lr=9.58e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  66%|██████▌   | 84/128 [04:16<01:53,  2.58s/it, pg=0.0968, rm=1.06, ret=-0.118, glen=168, tlen=874, kl=0.269, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  66%|██████▌   | 84/128 [04:18<01:53,  2.58s/it, pg=0.226, rm=0.688, ret=-0.244, glen=146, tlen=909, kl=0.24, act_lr=9.58e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  66%|██████▋   | 85/128 [04:18<01:41,  2.36s/it, pg=0.226, rm=0.688, ret=-0.244, glen=146, tlen=909, kl=0.24, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  66%|██████▋   | 85/128 [04:20<01:41,  2.36s/it, pg=-0.1, rm=1.19, ret=0.0795, glen=171, tlen=899, kl=0.252, act_lr=9.58e-7] 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  67%|██████▋   | 86/128 [04:20<01:37,  2.31s/it, pg=-0.1, rm=1.19, ret=0.0795, glen=171, tlen=899, kl=0.252, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  67%|██████▋   | 86/128 [04:22<01:37,  2.31s/it, pg=-0.0635, rm=0.875, ret=0.0454, glen=164, tlen=1135.75, kl=0.226, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  68%|██████▊   | 87/128 [04:22<01:31,  2.24s/it, pg=-0.0635, rm=0.875, ret=0.0454, glen=164, tlen=1135.75, kl=0.226, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  68%|██████▊   | 87/128 [04:32<01:31,  2.24s/it, pg=0.0884, rm=1.25, ret=-0.108, glen=149, tlen=742, kl=0.263, act_lr=9.58e-7]      
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  69%|██████▉   | 88/128 [04:32<02:54,  4.37s/it, pg=0.0884, rm=1.25, ret=-0.108, glen=149, tlen=742, kl=0.263, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  69%|██████▉   | 88/128 [04:34<02:54,  4.37s/it, pg=-0.116, rm=1.12, ret=0.0985, glen=144, tlen=952, kl=0.259, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  70%|██████▉   | 89/128 [04:34<02:23,  3.68s/it, pg=-0.116, rm=1.12, ret=0.0985, glen=144, tlen=952, kl=0.259, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  70%|██████▉   | 89/128 [04:36<02:23,  3.68s/it, pg=-0.0353, rm=1.25, ret=0.0177, glen=139, tlen=906, kl=0.256, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  70%|███████   | 90/128 [04:36<02:03,  3.24s/it, pg=-0.0353, rm=1.25, ret=0.0177, glen=139, tlen=906, kl=0.256, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  70%|███████   | 90/128 [04:39<02:03,  3.24s/it, pg=0.0697, rm=0.875, ret=-0.0864, glen=157, tlen=1.14e+3, kl=0.238, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  71%|███████   | 91/128 [04:39<01:50,  3.00s/it, pg=0.0697, rm=0.875, ret=-0.0864, glen=157, tlen=1.14e+3, kl=0.238, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  71%|███████   | 91/128 [04:41<01:50,  3.00s/it, pg=0.0844, rm=1, ret=-0.103, glen=155, tlen=907, kl=0.262, act_lr=9.58e-7]         
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  72%|███████▏  | 92/128 [04:41<01:37,  2.70s/it, pg=0.0844, rm=1, ret=-0.103, glen=155, tlen=907, kl=0.262, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  72%|███████▏  | 92/128 [04:43<01:37,  2.70s/it, pg=-0.0334, rm=1.25, ret=0.0154, glen=148, tlen=914, kl=0.256, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  73%|███████▎  | 93/128 [04:43<01:27,  2.49s/it, pg=-0.0334, rm=1.25, ret=0.0154, glen=148, tlen=914, kl=0.256, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  73%|███████▎  | 93/128 [04:45<01:27,  2.49s/it, pg=0.348, rm=0.688, ret=-0.366, glen=154, tlen=1.11e+3, kl=0.255, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  73%|███████▎  | 94/128 [04:45<01:22,  2.43s/it, pg=0.348, rm=0.688, ret=-0.366, glen=154, tlen=1.11e+3, kl=0.255, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  73%|███████▎  | 94/128 [04:47<01:22,  2.43s/it, pg=0.0889, rm=1.19, ret=-0.108, glen=145, tlen=756, kl=0.27, act_lr=9.58e-7]     
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  74%|███████▍  | 95/128 [04:47<01:13,  2.24s/it, pg=0.0889, rm=1.19, ret=-0.108, glen=145, tlen=756, kl=0.27, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  74%|███████▍  | 95/128 [04:56<01:13,  2.24s/it, pg=-0.111, rm=0.812, ret=0.0927, glen=155, tlen=918, kl=0.231, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  75%|███████▌  | 96/128 [04:56<02:20,  4.40s/it, pg=-0.111, rm=0.812, ret=0.0927, glen=155, tlen=918, kl=0.231, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  75%|███████▌  | 96/128 [04:58<02:20,  4.40s/it, pg=-0.0011, rm=1.12, ret=-0.0176, glen=154, tlen=1139.75, kl=0.254, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  76%|███████▌  | 97/128 [04:58<01:55,  3.72s/it, pg=-0.0011, rm=1.12, ret=-0.0176, glen=154, tlen=1139.75, kl=0.254, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  76%|███████▌  | 97/128 [05:00<01:55,  3.72s/it, pg=0.106, rm=1.06, ret=-0.125, glen=168, tlen=735, kl=0.247, act_lr=9.58e-7]       
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  77%|███████▋  | 98/128 [05:00<01:38,  3.28s/it, pg=0.106, rm=1.06, ret=-0.125, glen=168, tlen=735, kl=0.247, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  77%|███████▋  | 98/128 [05:02<01:38,  3.28s/it, pg=-0.28, rm=1, ret=0.262, glen=169, tlen=904, kl=0.25, act_lr=9.58e-7]     
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  77%|███████▋  | 99/128 [05:02<01:22,  2.86s/it, pg=-0.28, rm=1, ret=0.262, glen=169, tlen=904, kl=0.25, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  77%|███████▋  | 99/128 [05:04<01:22,  2.86s/it, pg=0.147, rm=1.06, ret=-0.167, glen=148, tlen=904, kl=0.282, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  78%|███████▊  | 100/128 [05:04<01:12,  2.60s/it, pg=0.147, rm=1.06, ret=-0.167, glen=148, tlen=904, kl=0.282, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  78%|███████▊  | 100/128 [05:07<01:12,  2.60s/it, pg=0.028, rm=0.875, ret=-0.0461, glen=148, tlen=987, kl=0.258, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  79%|███████▉  | 101/128 [05:07<01:07,  2.52s/it, pg=0.028, rm=0.875, ret=-0.0461, glen=148, tlen=987, kl=0.258, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  79%|███████▉  | 101/128 [05:09<01:07,  2.52s/it, pg=0.115, rm=1.12, ret=-0.132, glen=161, tlen=818, kl=0.254, act_lr=9.58e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  80%|███████▉  | 102/128 [05:09<01:02,  2.39s/it, pg=0.115, rm=1.12, ret=-0.132, glen=161, tlen=818, kl=0.254, act_lr=9.58e-7]
+(ActorModelRayActor pid=430510) 
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  80%|███████▉  | 102/128 [05:11<01:02,  2.39s/it, pg=-0.0809, rm=1, ret=0.062, glen=160, tlen=943, kl=0.249, act_lr=9.58e-7]  
+(ActorModelRayActor pid=430510) 
+Train epoch [1/1]:  80%|████████  | 103/128 [05:11<00:57,  2.31s/it, pg=-0.0809, rm=1, ret=0.062, glen=160, tlen=943, kl=0.249, act_lr=9.58e-7]
+(raylet) A worker died or was killed while executing a task by an unexpected system error. To troubleshoot the problem, check the logs for the dead worker. RayTask ID: ffffffffffffffffea4638014719d506e52c460502000000 Worker ID: c0d43efe95f45b4e9963692e567d519faa12d6a914ec86da31d0012a Node ID: 540befe396d93398a15226f02194a4239b64f9b59ae8b6f86ede4d86 Worker IP address: 10.140.0.151 Worker port: 20133 Worker PID: 431335 Worker exit type: SYSTEM_ERROR Worker exit detail: Worker unexpectedly exits with a connection error code 2. End of file. There are some potential root causes. (1) The process is killed by SIGKILL by OOM killer due to high memory usage. (2) ray stop --force is called. (3) The worker is crashed unexpectedly due to SIGSEGV or other unexpected errors.
+Traceback (most recent call last):
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/connection.py", line 198, in _new_conn
+    sock = connection.create_connection(
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection
+    raise err
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/util/connection.py", line 73, in create_connection
+    sock.connect(sa)
+ConnectionRefusedError: [Errno 111] Connection refused
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/connectionpool.py", line 787, in urlopen
+    response = self._make_request(
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/connectionpool.py", line 493, in _make_request
+    conn.request(
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/connection.py", line 445, in request
+    self.endheaders()
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/http/client.py", line 1278, in endheaders
+    self._send_output(message_body, encode_chunked=encode_chunked)
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/http/client.py", line 1038, in _send_output
+    self.send(msg)
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/http/client.py", line 976, in send
+    self.connect()
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/connection.py", line 276, in connect
+    self.sock = self._new_conn()
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/connection.py", line 213, in _new_conn
+    raise NewConnectionError(
+urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7fc5d6a86470>: Failed to establish a new connection: [Errno 111] Connection refused
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/requests/adapters.py", line 667, in send
+    resp = conn.urlopen(
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/connectionpool.py", line 841, in urlopen
+    retries = retries.increment(
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/urllib3/util/retry.py", line 519, in increment
+    raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]
+urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='127.0.0.1', port=2989): Max retries exceeded with url: /api/jobs/raysubmit_tdQtQ5pUh6YdmvqQ (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fc5d6a86470>: Failed to establish a new connection: [Errno 111] Connection refused'))
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/bin/ray", line 8, in <module>
+    sys.exit(main())
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/ray/scripts/scripts.py", line 2690, in main
+    return cli()
+  File "/mnt/petrelfs/luyiting/anaconda3/envs/lmmr1/lib/python3.10/site-packages/click/core.py", line 1161, in __call__