Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +0 -0
- debug-internal.log +32 -0
- debug.log +27 -0
- run-20260606_101540-71wbmn3f/files/config.yaml +39 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_0cb1045c40045005886e.png +0 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_1f7b70d4340f1682d116.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_243c70f367b210f65092.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_3b28a4e69bd3270b76ad.png +0 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_43cdd7a37f85c6a13c2e.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_66feaebf5350fc850b80.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_6f4aa734b7c88a6d9df9.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_78450a1ff8752a736686.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_97c186f4c58d304f4e76.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_ac50aa2464a6912a75bf.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_ac7ce769f360461ad2ea.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_d261a691537a4ec6a12b.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_d4e7c1934d99188c6708.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_d81fc34b4e3bc9379445.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_fd4bdd1ff66d8ba33988.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/images_0_0a77a4658bd6cc6fca47.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/images_0_2b1e366339eedfbce997.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/images_0_50d936f2e6f165ae7ff7.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/images_0_605692edaa37b67dc68a.png +0 -0
- run-20260606_101540-71wbmn3f/files/media/images/images_0_90e8321e45af352448c5.png +0 -0
- run-20260606_101540-71wbmn3f/files/media/images/images_0_9e14dcb20935e442f24d.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/images_0_bc6a36f25e0e0cf9631f.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/images_0_ecd25b0e9060ac0b15cf.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_2296d182e5a0026c5b49.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_518103ac97c84ab1036d.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_6c599567e8776dc5cbf9.png +0 -0
- run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_875c04962a4a6cb88d52.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_88a3182e8d6a3b61e4c3.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_db13edf7b89f80a636f2.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_f88f9c1b09e0d91499c1.png +3 -0
- run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_feb9cce0d402d0bdf356.png +0 -0
- run-20260606_101540-71wbmn3f/files/output.log +685 -0
- run-20260606_101540-71wbmn3f/files/wandb-metadata.json +90 -0
- run-20260606_101540-71wbmn3f/files/wandb-summary.json +1 -0
- run-20260606_101540-71wbmn3f/logs/debug-internal.log +16 -0
- run-20260606_101540-71wbmn3f/logs/debug.log +27 -0
- run-20260606_101540-71wbmn3f/run-71wbmn3f.wandb +3 -0
- run-20260606_110607-g0uqiw13/files/config.yaml +39 -0
- run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_0cb1045c40045005886e.png +0 -0
- run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_1f7b70d4340f1682d116.png +3 -0
- run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_243c70f367b210f65092.png +3 -0
- run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_3b28a4e69bd3270b76ad.png +0 -0
- run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_43cdd7a37f85c6a13c2e.png +3 -0
- run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_66feaebf5350fc850b80.png +3 -0
- run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_6f4aa734b7c88a6d9df9.png +3 -0
- run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_78450a1ff8752a736686.png +3 -0
.gitattributes
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
debug-internal.log
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-06-08T22:39:06.92175061+08:00","level":"INFO","msg":"using version","core version":"0.18.7"}
|
| 2 |
+
{"time":"2026-06-08T22:39:06.921886628+08:00","level":"INFO","msg":"created symlink","path":"/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/wandb/run-20260608_223906-gg4s7434/logs/debug-core.log"}
|
| 3 |
+
{"time":"2026-06-08T22:39:07.134578244+08:00","level":"INFO","msg":"created new stream","id":"gg4s7434"}
|
| 4 |
+
{"time":"2026-06-08T22:39:07.134729721+08:00","level":"INFO","msg":"stream: started","id":"gg4s7434"}
|
| 5 |
+
{"time":"2026-06-08T22:39:07.134818+08:00","level":"INFO","msg":"sender: started","stream_id":"gg4s7434"}
|
| 6 |
+
{"time":"2026-06-08T22:39:07.134770161+08:00","level":"INFO","msg":"handler: started","stream_id":"gg4s7434"}
|
| 7 |
+
{"time":"2026-06-08T22:39:07.134762901+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"gg4s7434"}
|
| 8 |
+
{"time":"2026-06-08T22:39:08.317564647+08:00","level":"INFO","msg":"Starting system monitor"}
|
| 9 |
+
{"time":"2026-06-08T22:52:32.380561827+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 10 |
+
{"time":"2026-06-09T00:53:09.800439559+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 11 |
+
{"time":"2026-06-09T01:18:40.473570759+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 12 |
+
{"time":"2026-06-09T02:27:02.663288567+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 13 |
+
{"time":"2026-06-09T03:56:30.277379071+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 14 |
+
{"time":"2026-06-09T04:36:19.171312261+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 15 |
+
{"time":"2026-06-09T05:55:18.844598137+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 16 |
+
{"time":"2026-06-09T10:32:45.267536169+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 17 |
+
{"time":"2026-06-09T10:49:57.6770408+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 18 |
+
{"time":"2026-06-09T11:00:48.052936723+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 19 |
+
{"time":"2026-06-09T12:34:10.241294508+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 20 |
+
{"time":"2026-06-09T13:15:36.270900225+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 21 |
+
{"time":"2026-06-09T13:22:04.952465805+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 22 |
+
{"time":"2026-06-09T15:07:33.565240216+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 23 |
+
{"time":"2026-06-09T15:50:00.534354241+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 24 |
+
{"time":"2026-06-09T16:44:59.012610896+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/1341610137-xi-an-jiaotong-university-/fixwin_h20/gg4s7434/file_stream\": unexpected EOF"}
|
| 25 |
+
{"time":"2026-06-09T18:35:37.160915696+08:00","level":"INFO","msg":"stream: closing","id":"gg4s7434"}
|
| 26 |
+
{"time":"2026-06-09T18:35:37.161000285+08:00","level":"INFO","msg":"Stopping system monitor"}
|
| 27 |
+
{"time":"2026-06-09T18:35:37.162197787+08:00","level":"INFO","msg":"Stopped system monitor"}
|
| 28 |
+
{"time":"2026-06-09T18:35:41.835551663+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 29 |
+
{"time":"2026-06-09T18:35:42.577370328+08:00","level":"INFO","msg":"handler: closed","stream_id":"gg4s7434"}
|
| 30 |
+
{"time":"2026-06-09T18:35:42.577444966+08:00","level":"INFO","msg":"sender: closed","stream_id":"gg4s7434"}
|
| 31 |
+
{"time":"2026-06-09T18:35:42.577436277+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"gg4s7434"}
|
| 32 |
+
{"time":"2026-06-09T18:35:42.578387762+08:00","level":"INFO","msg":"stream: closed","id":"gg4s7434"}
|
debug.log
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-06-08 22:39:06,904 INFO MainThread:493388 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
|
| 2 |
+
2026-06-08 22:39:06,905 INFO MainThread:493388 [wandb_setup.py:_flush():79] Configure stats pid to 493388
|
| 3 |
+
2026-06-08 22:39:06,905 INFO MainThread:493388 [wandb_setup.py:_flush():79] Loading settings from /root/.config/wandb/settings
|
| 4 |
+
2026-06-08 22:39:06,905 INFO MainThread:493388 [wandb_setup.py:_flush():79] Loading settings from /apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/wandb/settings
|
| 5 |
+
2026-06-08 22:39:06,905 WARNING MainThread:493388 [wandb_setup.py:_flush():79] Unknown environment variable: WANDB_GROUP
|
| 6 |
+
2026-06-08 22:39:06,905 INFO MainThread:493388 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'project': 'fixwin_h20', 'api_key': '***REDACTED***', 'resume': 'never', 'run_name': 'statcm_win1_10full_n001_bs64_h20', 'mode': 'online', 'run_tags': ['img-only', 'scratch30k', 'manual', 'noise0.01', 'bs64']}
|
| 7 |
+
2026-06-08 22:39:06,905 INFO MainThread:493388 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
|
| 8 |
+
2026-06-08 22:39:06,905 INFO MainThread:493388 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'scripts/train_omnigen.py', 'program_abspath': '/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/scripts/train_omnigen.py', 'program': '/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/scripts/train_omnigen.py'}
|
| 9 |
+
2026-06-08 22:39:06,905 INFO MainThread:493388 [wandb_setup.py:_flush():79] Applying login settings: {}
|
| 10 |
+
2026-06-08 22:39:06,906 INFO MainThread:493388 [wandb_init.py:_log_setup():533] Logging user logs to /apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/wandb/run-20260608_223906-gg4s7434/logs/debug.log
|
| 11 |
+
2026-06-08 22:39:06,907 INFO MainThread:493388 [wandb_init.py:_log_setup():534] Logging internal logs to /apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/wandb/run-20260608_223906-gg4s7434/logs/debug-internal.log
|
| 12 |
+
2026-06-08 22:39:06,907 INFO MainThread:493388 [wandb_init.py:init():619] calling init triggers
|
| 13 |
+
2026-06-08 22:39:06,907 INFO MainThread:493388 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
|
| 14 |
+
config: {}
|
| 15 |
+
2026-06-08 22:39:06,907 INFO MainThread:493388 [wandb_init.py:init():669] starting backend
|
| 16 |
+
2026-06-08 22:39:06,907 INFO MainThread:493388 [wandb_init.py:init():673] sending inform_init request
|
| 17 |
+
2026-06-08 22:39:06,918 INFO MainThread:493388 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
| 18 |
+
2026-06-08 22:39:06,919 INFO MainThread:493388 [wandb_init.py:init():686] backend started and connected
|
| 19 |
+
2026-06-08 22:39:06,942 INFO MainThread:493388 [wandb_init.py:init():781] updated telemetry
|
| 20 |
+
2026-06-08 22:39:07,017 INFO MainThread:493388 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
|
| 21 |
+
2026-06-08 22:39:08,295 INFO MainThread:493388 [wandb_init.py:init():867] starting run threads in backend
|
| 22 |
+
2026-06-08 22:39:08,642 INFO MainThread:493388 [wandb_run.py:_console_start():2456] atexit reg
|
| 23 |
+
2026-06-08 22:39:08,642 INFO MainThread:493388 [wandb_run.py:_redirect():2305] redirect: wrap_raw
|
| 24 |
+
2026-06-08 22:39:08,643 INFO MainThread:493388 [wandb_run.py:_redirect():2370] Wrapping output streams.
|
| 25 |
+
2026-06-08 22:39:08,643 INFO MainThread:493388 [wandb_run.py:_redirect():2395] Redirects installed.
|
| 26 |
+
2026-06-08 22:39:08,666 INFO MainThread:493388 [wandb_init.py:init():911] run started, returning control to user process
|
| 27 |
+
2026-06-09 18:35:37,161 WARNING MsgRouterThr:493388 [router.py:message_loop():75] message_loop has been closed
|
run-20260606_101540-71wbmn3f/files/config.yaml
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.18.7
|
| 4 |
+
m: []
|
| 5 |
+
python_version: 3.10.20
|
| 6 |
+
t:
|
| 7 |
+
"1":
|
| 8 |
+
- 1
|
| 9 |
+
- 11
|
| 10 |
+
- 41
|
| 11 |
+
- 49
|
| 12 |
+
- 55
|
| 13 |
+
- 63
|
| 14 |
+
- 71
|
| 15 |
+
- 83
|
| 16 |
+
- 98
|
| 17 |
+
"2":
|
| 18 |
+
- 1
|
| 19 |
+
- 11
|
| 20 |
+
- 41
|
| 21 |
+
- 49
|
| 22 |
+
- 53
|
| 23 |
+
- 55
|
| 24 |
+
- 63
|
| 25 |
+
- 71
|
| 26 |
+
- 83
|
| 27 |
+
- 98
|
| 28 |
+
"3":
|
| 29 |
+
- 13
|
| 30 |
+
- 23
|
| 31 |
+
- 55
|
| 32 |
+
- 61
|
| 33 |
+
"4": 3.10.20
|
| 34 |
+
"5": 0.18.7
|
| 35 |
+
"6": 4.45.2
|
| 36 |
+
"8":
|
| 37 |
+
- 5
|
| 38 |
+
"12": 0.18.7
|
| 39 |
+
"13": linux-x86_64
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_0cb1045c40045005886e.png
ADDED
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_1f7b70d4340f1682d116.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_243c70f367b210f65092.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_3b28a4e69bd3270b76ad.png
ADDED
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_43cdd7a37f85c6a13c2e.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_66feaebf5350fc850b80.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_6f4aa734b7c88a6d9df9.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_78450a1ff8752a736686.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_97c186f4c58d304f4e76.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_ac50aa2464a6912a75bf.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_ac7ce769f360461ad2ea.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_d261a691537a4ec6a12b.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_d4e7c1934d99188c6708.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_d81fc34b4e3bc9379445.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/eval_images_0_fd4bdd1ff66d8ba33988.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/images_0_0a77a4658bd6cc6fca47.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/images_0_2b1e366339eedfbce997.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/images_0_50d936f2e6f165ae7ff7.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/images_0_605692edaa37b67dc68a.png
ADDED
|
run-20260606_101540-71wbmn3f/files/media/images/images_0_90e8321e45af352448c5.png
ADDED
|
run-20260606_101540-71wbmn3f/files/media/images/images_0_9e14dcb20935e442f24d.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/images_0_bc6a36f25e0e0cf9631f.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/images_0_ecd25b0e9060ac0b15cf.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_2296d182e5a0026c5b49.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_518103ac97c84ab1036d.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_6c599567e8776dc5cbf9.png
ADDED
|
run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_875c04962a4a6cb88d52.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_88a3182e8d6a3b61e4c3.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_db13edf7b89f80a636f2.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_f88f9c1b09e0d91499c1.png
ADDED
|
Git LFS Details
|
run-20260606_101540-71wbmn3f/files/media/images/train_rollout_images_0_feb9cce0d402d0bdf356.png
ADDED
|
run-20260606_101540-71wbmn3f/files/output.log
ADDED
|
@@ -0,0 +1,685 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
I0606 10:15:43.189767 140545645860672 train_omnigen.py:922] Resolved joint config summary: max_eval_batches=2, eval_num_steps=50, num_steps=10, noise_level=0.01, train_batch_size=32, test_batch_size=32, num_image_per_prompt=4, beta=0.005
|
| 2 |
+
Fetching 10 files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [25:51<00:00, 155.12s/it]
|
| 3 |
+
Loading safetensors
|
| 4 |
+
I0606 10:42:17.383409 140545645860672 train_omnigen.py:625] Merging OmniGen LoRA into base weights from /apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/gen_joint/results_new/scratch_15k
|
| 5 |
+
/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/peft/tuners/tuners_utils.py:285: UserWarning: Already found a `peft_config` attribute in the model. This will lead to having multiple adapters in the model. Make sure to know what you are doing!
|
| 6 |
+
warnings.warn(
|
| 7 |
+
I0606 10:42:19.604990 140545645860672 train_omnigen.py:1003] Model/component load finished in 1596.4s
|
| 8 |
+
I0606 10:42:36.062630 140545645860672 train_omnigen.py:1119] ***** Running OmniGen GRPO training *****
|
| 9 |
+
I0606 10:42:36.063020 140545645860672 train_omnigen.py:1120] Finetuning mode = LoRA
|
| 10 |
+
I0606 10:42:36.063105 140545645860672 train_omnigen.py:1122] LoRA rank / alpha = 32 / 64
|
| 11 |
+
I0606 10:42:36.063339 140545645860672 train_omnigen.py:1127] Merged base LoRA path = /apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/gen_joint/results_new/scratch_15k
|
| 12 |
+
I0606 10:42:36.063584 140545645860672 train_omnigen.py:1128] Trainable LoRA path = None
|
| 13 |
+
I0606 10:42:36.063688 140545645860672 train_omnigen.py:1129] KL reference mode = base_model_without_adapter
|
| 14 |
+
I0606 10:42:36.064837 140545645860672 train_omnigen.py:1131] Trainable parameters = 18,874,368 / 3,780,946,960 (0.50%); trainable tensors=128; LoRA targets=['qkv_proj', 'o_proj']
|
| 15 |
+
I0606 10:42:36.064933 140545645860672 train_omnigen.py:1141] Sample batch size per device = 32
|
| 16 |
+
I0606 10:42:36.065003 140545645860672 train_omnigen.py:1142] Train batch size per device = 32
|
| 17 |
+
I0606 10:42:36.065060 140545645860672 train_omnigen.py:1143] Gradient accumulation steps = 1
|
| 18 |
+
I0606 10:42:36.065113 140545645860672 train_omnigen.py:1144] Samples per epoch = 256
|
| 19 |
+
I0606 10:42:36.065161 140545645860672 train_omnigen.py:1145] Effective train batch size = 128
|
| 20 |
+
I0606 10:42:36.067547 140545645860672 train_omnigen.py:762] Starting train-time Eval: this is no-grad monitoring, not optimizer training. max_eval_batches=2, eval_num_steps=50, eval_noise_level=0.0, eval_batches=2, eval_dataset_size=23984
|
| 21 |
+
Eval: 0%| | 0/2 [00:00<?, ?it/s]I0606 10:46:14.461350 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 22 |
+
I0606 10:46:14.578375 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0001.
|
| 23 |
+
I0606 10:46:14.729732 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 24 |
+
I0606 10:46:14.730793 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0001_pred.
|
| 25 |
+
I0606 10:46:14.810051 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 26 |
+
I0606 10:46:14.811182 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0002.
|
| 27 |
+
I0606 10:46:14.862700 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 28 |
+
I0606 10:46:14.863620 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0002_pred.
|
| 29 |
+
I0606 10:46:14.940365 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 30 |
+
I0606 10:46:14.941375 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0003.
|
| 31 |
+
I0606 10:46:14.992277 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 32 |
+
I0606 10:46:14.993198 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0003_pred.
|
| 33 |
+
I0606 10:46:15.068730 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 34 |
+
I0606 10:46:15.069696 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0004.
|
| 35 |
+
I0606 10:46:15.122958 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 36 |
+
I0606 10:46:15.123900 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0004_pred.
|
| 37 |
+
I0606 10:46:15.206114 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 38 |
+
I0606 10:46:15.207276 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0005.
|
| 39 |
+
I0606 10:46:15.258195 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 40 |
+
I0606 10:46:15.259088 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0005_pred.
|
| 41 |
+
I0606 10:46:15.338641 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 42 |
+
I0606 10:46:15.339638 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0006.
|
| 43 |
+
I0606 10:46:15.391009 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 44 |
+
I0606 10:46:15.392149 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0006_pred.
|
| 45 |
+
I0606 10:46:15.470912 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 46 |
+
I0606 10:46:15.471944 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0007.
|
| 47 |
+
I0606 10:46:15.523969 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 48 |
+
I0606 10:46:15.524984 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0007_pred.
|
| 49 |
+
I0606 10:46:15.605497 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 50 |
+
I0606 10:46:15.606590 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0008.
|
| 51 |
+
I0606 10:46:15.657006 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 52 |
+
I0606 10:46:15.657915 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0008_pred.
|
| 53 |
+
I0606 10:46:15.739175 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 54 |
+
I0606 10:46:15.740217 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0009.
|
| 55 |
+
I0606 10:46:15.791176 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 56 |
+
I0606 10:46:15.792084 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0009_pred.
|
| 57 |
+
I0606 10:46:15.873632 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 58 |
+
I0606 10:46:15.874666 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0010.
|
| 59 |
+
I0606 10:46:15.925530 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 60 |
+
I0606 10:46:15.926476 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0010_pred.
|
| 61 |
+
I0606 10:46:16.011161 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 62 |
+
I0606 10:46:16.012387 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0011.
|
| 63 |
+
I0606 10:46:16.064039 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 64 |
+
I0606 10:46:16.064983 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0011_pred.
|
| 65 |
+
I0606 10:46:16.146904 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 66 |
+
I0606 10:46:16.147936 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0012.
|
| 67 |
+
I0606 10:46:16.199387 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 68 |
+
I0606 10:46:16.200327 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0012_pred.
|
| 69 |
+
I0606 10:46:16.280089 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 70 |
+
I0606 10:46:16.281117 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0013.
|
| 71 |
+
I0606 10:46:16.331746 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 72 |
+
I0606 10:46:16.332667 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0013_pred.
|
| 73 |
+
I0606 10:46:16.417457 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 74 |
+
I0606 10:46:16.418497 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0014.
|
| 75 |
+
I0606 10:46:16.468961 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 76 |
+
I0606 10:46:16.469825 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0014_pred.
|
| 77 |
+
I0606 10:46:16.550191 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 78 |
+
I0606 10:46:16.551213 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0015.
|
| 79 |
+
I0606 10:46:16.606454 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 80 |
+
I0606 10:46:16.607573 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0015_pred.
|
| 81 |
+
I0606 10:46:16.684311 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 82 |
+
I0606 10:46:16.685323 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0016.
|
| 83 |
+
I0606 10:46:16.736419 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 84 |
+
I0606 10:46:16.737384 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0016_pred.
|
| 85 |
+
I0606 10:46:16.817455 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 86 |
+
I0606 10:46:16.818640 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0017.
|
| 87 |
+
I0606 10:46:16.870932 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 88 |
+
I0606 10:46:16.871890 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0017_pred.
|
| 89 |
+
I0606 10:46:16.950687 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 90 |
+
I0606 10:46:16.951751 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0018.
|
| 91 |
+
I0606 10:46:17.006537 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 92 |
+
I0606 10:46:17.007583 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0018_pred.
|
| 93 |
+
I0606 10:46:17.078089 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 94 |
+
I0606 10:46:17.079117 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0019.
|
| 95 |
+
I0606 10:46:17.130991 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 96 |
+
I0606 10:46:17.132033 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0019_pred.
|
| 97 |
+
I0606 10:46:17.203898 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 98 |
+
I0606 10:46:17.205075 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0020.
|
| 99 |
+
I0606 10:46:17.257932 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 100 |
+
I0606 10:46:17.258959 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0020_pred.
|
| 101 |
+
I0606 10:46:17.338996 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 102 |
+
I0606 10:46:17.340086 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0021.
|
| 103 |
+
I0606 10:46:17.391327 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 104 |
+
I0606 10:46:17.392431 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0021_pred.
|
| 105 |
+
I0606 10:46:17.469836 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 106 |
+
I0606 10:46:17.470891 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0022.
|
| 107 |
+
I0606 10:46:17.525337 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 108 |
+
I0606 10:46:17.526563 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0022_pred.
|
| 109 |
+
I0606 10:46:17.602098 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 110 |
+
I0606 10:46:17.603183 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0023.
|
| 111 |
+
I0606 10:46:17.655370 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 112 |
+
I0606 10:46:17.656404 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0023_pred.
|
| 113 |
+
I0606 10:46:17.734316 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 114 |
+
I0606 10:46:17.735466 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0024.
|
| 115 |
+
I0606 10:46:17.786669 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 116 |
+
I0606 10:46:17.787603 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0024_pred.
|
| 117 |
+
I0606 10:46:17.865211 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 118 |
+
I0606 10:46:17.866264 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0025.
|
| 119 |
+
I0606 10:46:17.918962 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 120 |
+
I0606 10:46:17.920056 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0025_pred.
|
| 121 |
+
I0606 10:46:17.996622 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 122 |
+
I0606 10:46:17.997730 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0026.
|
| 123 |
+
I0606 10:46:18.052341 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 124 |
+
I0606 10:46:18.053373 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0026_pred.
|
| 125 |
+
I0606 10:46:18.132566 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 126 |
+
I0606 10:46:18.133712 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0027.
|
| 127 |
+
I0606 10:46:18.188274 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 128 |
+
I0606 10:46:18.189730 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0027_pred.
|
| 129 |
+
I0606 10:46:18.267812 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 130 |
+
I0606 10:46:18.268868 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0028.
|
| 131 |
+
I0606 10:46:18.320708 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 132 |
+
I0606 10:46:18.321769 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0028_pred.
|
| 133 |
+
I0606 10:46:18.402666 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 134 |
+
I0606 10:46:18.403806 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0029.
|
| 135 |
+
I0606 10:46:18.455156 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 136 |
+
I0606 10:46:18.456184 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0029_pred.
|
| 137 |
+
I0606 10:46:18.534491 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 138 |
+
I0606 10:46:18.535603 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0030.
|
| 139 |
+
I0606 10:46:18.586899 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 140 |
+
I0606 10:46:18.587900 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0030_pred.
|
| 141 |
+
I0606 10:46:18.663362 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 142 |
+
I0606 10:46:18.664427 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0031.
|
| 143 |
+
I0606 10:46:18.715521 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 144 |
+
I0606 10:46:18.716575 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0031_pred.
|
| 145 |
+
I0606 10:46:18.791826 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 146 |
+
I0606 10:46:18.792884 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0032.
|
| 147 |
+
I0606 10:46:18.847477 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 148 |
+
I0606 10:46:18.848518 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0032_pred.
|
| 149 |
+
Eval: 50%|█████████████████████████████████████████████████████████ | 1/2 [03:43<03:43, 223.93s/it]I0606 10:49:16.487822 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 150 |
+
I0606 10:49:16.489390 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0129.
|
| 151 |
+
I0606 10:49:16.546560 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 152 |
+
I0606 10:49:16.547580 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0129_pred.
|
| 153 |
+
I0606 10:49:16.627927 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 154 |
+
I0606 10:49:16.628995 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0130.
|
| 155 |
+
I0606 10:49:16.679118 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 156 |
+
I0606 10:49:16.680032 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0130_pred.
|
| 157 |
+
I0606 10:49:16.761204 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 158 |
+
I0606 10:49:16.762253 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0131.
|
| 159 |
+
I0606 10:49:16.817443 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 160 |
+
I0606 10:49:16.818479 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0131_pred.
|
| 161 |
+
I0606 10:49:16.894864 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 162 |
+
I0606 10:49:16.896259 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0132.
|
| 163 |
+
I0606 10:49:16.951329 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 164 |
+
I0606 10:49:16.952363 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0132_pred.
|
| 165 |
+
I0606 10:49:17.040236 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 166 |
+
I0606 10:49:17.041291 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0133.
|
| 167 |
+
I0606 10:49:17.093331 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 168 |
+
I0606 10:49:17.094653 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0133_pred.
|
| 169 |
+
I0606 10:49:17.176516 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 170 |
+
I0606 10:49:17.177565 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0134.
|
| 171 |
+
I0606 10:49:17.235712 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 172 |
+
I0606 10:49:17.236735 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0134_pred.
|
| 173 |
+
I0606 10:49:17.316303 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 174 |
+
I0606 10:49:17.317415 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0135.
|
| 175 |
+
I0606 10:49:17.367291 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 176 |
+
I0606 10:49:17.368199 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0135_pred.
|
| 177 |
+
I0606 10:49:17.448949 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 178 |
+
I0606 10:49:17.449992 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0136.
|
| 179 |
+
I0606 10:49:17.508253 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 180 |
+
I0606 10:49:17.509483 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0136_pred.
|
| 181 |
+
I0606 10:49:17.585163 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 182 |
+
I0606 10:49:17.586178 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0137.
|
| 183 |
+
I0606 10:49:17.642778 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 184 |
+
I0606 10:49:17.643800 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0137_pred.
|
| 185 |
+
I0606 10:49:17.723769 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 186 |
+
I0606 10:49:17.724864 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0138.
|
| 187 |
+
I0606 10:49:17.775604 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 188 |
+
I0606 10:49:17.776524 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0138_pred.
|
| 189 |
+
I0606 10:49:17.860145 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 190 |
+
I0606 10:49:17.861193 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0139.
|
| 191 |
+
I0606 10:49:17.918184 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 192 |
+
I0606 10:49:17.919217 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0139_pred.
|
| 193 |
+
I0606 10:49:17.995276 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 194 |
+
I0606 10:49:17.996682 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0140.
|
| 195 |
+
I0606 10:49:18.048686 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 196 |
+
I0606 10:49:18.049670 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0140_pred.
|
| 197 |
+
I0606 10:49:18.133312 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 198 |
+
I0606 10:49:18.134387 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0141.
|
| 199 |
+
I0606 10:49:18.183904 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 200 |
+
I0606 10:49:18.184813 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0141_pred.
|
| 201 |
+
I0606 10:49:18.268893 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 202 |
+
I0606 10:49:18.269943 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0142.
|
| 203 |
+
I0606 10:49:18.329229 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 204 |
+
I0606 10:49:18.330265 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0142_pred.
|
| 205 |
+
I0606 10:49:18.412131 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 206 |
+
I0606 10:49:18.413265 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0143.
|
| 207 |
+
I0606 10:49:18.467186 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 208 |
+
I0606 10:49:18.468117 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0143_pred.
|
| 209 |
+
I0606 10:49:18.555353 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 210 |
+
I0606 10:49:18.556551 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0144.
|
| 211 |
+
I0606 10:49:18.615081 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 212 |
+
I0606 10:49:18.616135 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0144_pred.
|
| 213 |
+
I0606 10:49:18.699170 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 214 |
+
I0606 10:49:18.700295 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0145.
|
| 215 |
+
I0606 10:49:18.751366 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 216 |
+
I0606 10:49:18.752393 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0145_pred.
|
| 217 |
+
I0606 10:49:18.833409 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 218 |
+
I0606 10:49:18.834480 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0146.
|
| 219 |
+
I0606 10:49:18.884959 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 220 |
+
I0606 10:49:18.885852 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0146_pred.
|
| 221 |
+
I0606 10:49:18.968161 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 222 |
+
I0606 10:49:18.969210 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0147.
|
| 223 |
+
I0606 10:49:19.020893 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 224 |
+
I0606 10:49:19.021929 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0147_pred.
|
| 225 |
+
I0606 10:49:19.106423 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 226 |
+
I0606 10:49:19.108144 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0148.
|
| 227 |
+
I0606 10:49:19.159424 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 228 |
+
I0606 10:49:19.160412 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0148_pred.
|
| 229 |
+
I0606 10:49:19.241507 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 230 |
+
I0606 10:49:19.242572 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0149.
|
| 231 |
+
I0606 10:49:19.293288 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 232 |
+
I0606 10:49:19.294343 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0149_pred.
|
| 233 |
+
I0606 10:49:19.375447 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 234 |
+
I0606 10:49:19.376515 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0150.
|
| 235 |
+
I0606 10:49:19.428834 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 236 |
+
I0606 10:49:19.429877 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0150_pred.
|
| 237 |
+
I0606 10:49:19.513043 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 238 |
+
I0606 10:49:19.514170 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0151.
|
| 239 |
+
I0606 10:49:19.564903 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 240 |
+
I0606 10:49:19.565809 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0151_pred.
|
| 241 |
+
I0606 10:49:19.654678 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 242 |
+
I0606 10:49:19.655714 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0152.
|
| 243 |
+
I0606 10:49:19.713363 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 244 |
+
I0606 10:49:19.714429 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0152_pred.
|
| 245 |
+
I0606 10:49:19.787099 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 246 |
+
I0606 10:49:19.788427 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0153.
|
| 247 |
+
I0606 10:49:19.844462 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 248 |
+
I0606 10:49:19.845476 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0153_pred.
|
| 249 |
+
I0606 10:49:19.924469 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 250 |
+
I0606 10:49:19.925552 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0154.
|
| 251 |
+
I0606 10:49:19.974787 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 252 |
+
I0606 10:49:19.975686 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0154_pred.
|
| 253 |
+
I0606 10:49:20.056912 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 254 |
+
I0606 10:49:20.057949 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0155.
|
| 255 |
+
I0606 10:49:20.115859 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 256 |
+
I0606 10:49:20.116901 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0155_pred.
|
| 257 |
+
I0606 10:49:20.191797 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 258 |
+
I0606 10:49:20.193148 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0156.
|
| 259 |
+
I0606 10:49:20.248276 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 260 |
+
I0606 10:49:20.249279 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0156_pred.
|
| 261 |
+
I0606 10:49:20.332039 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 262 |
+
I0606 10:49:20.333104 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0157.
|
| 263 |
+
I0606 10:49:20.383141 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 264 |
+
I0606 10:49:20.384045 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0157_pred.
|
| 265 |
+
I0606 10:49:20.468678 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 266 |
+
I0606 10:49:20.469727 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0158.
|
| 267 |
+
I0606 10:49:20.527606 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 268 |
+
I0606 10:49:20.528654 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0158_pred.
|
| 269 |
+
I0606 10:49:20.608254 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 270 |
+
I0606 10:49:20.609526 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0159.
|
| 271 |
+
I0606 10:49:20.660390 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 272 |
+
I0606 10:49:20.661328 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0159_pred.
|
| 273 |
+
I0606 10:49:20.744742 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 274 |
+
I0606 10:49:20.745779 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0160.
|
| 275 |
+
I0606 10:49:20.801387 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 276 |
+
I0606 10:49:20.802748 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0160_pred.
|
| 277 |
+
Eval: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [06:45<00:00, 202.76s/it]
|
| 278 |
+
I0606 10:49:23.531781 140545645860672 train_omnigen.py:875] Train-time Eval finished in 407.5s
|
| 279 |
+
I0606 10:49:23.532423 140545645860672 train_omnigen.py:1179] Starting no-grad rollout sampling for epoch 0
|
| 280 |
+
Epoch 0: sampling: 50%|███████████████████████████████████████████████████ | 1/2 [00:36<00:36, 36.94s/it]I0606 10:50:00.954293 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 281 |
+
I0606 10:50:00.955646 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1086.
|
| 282 |
+
I0606 10:50:01.016146 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 283 |
+
I0606 10:50:01.017235 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1086_pred.
|
| 284 |
+
I0606 10:50:01.101151 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 285 |
+
I0606 10:50:01.102696 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0939.
|
| 286 |
+
I0606 10:50:01.153737 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 287 |
+
I0606 10:50:01.154734 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0939_pred.
|
| 288 |
+
I0606 10:50:01.239955 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 289 |
+
I0606 10:50:01.241308 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0208.
|
| 290 |
+
I0606 10:50:01.306994 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 291 |
+
I0606 10:50:01.308485 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0208_pred.
|
| 292 |
+
I0606 10:50:01.399291 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 293 |
+
I0606 10:50:01.400970 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0481.
|
| 294 |
+
I0606 10:50:01.462644 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 295 |
+
I0606 10:50:01.463660 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0481_pred.
|
| 296 |
+
I0606 10:50:01.556651 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 297 |
+
I0606 10:50:01.557698 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0630.
|
| 298 |
+
I0606 10:50:01.617092 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 299 |
+
I0606 10:50:01.618927 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0630_pred.
|
| 300 |
+
I0606 10:50:01.696538 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 301 |
+
I0606 10:50:01.697889 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0683.
|
| 302 |
+
I0606 10:50:01.747540 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 303 |
+
I0606 10:50:01.748458 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0683_pred.
|
| 304 |
+
I0606 10:50:01.827553 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 305 |
+
I0606 10:50:01.828620 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1483.
|
| 306 |
+
I0606 10:50:01.875881 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 307 |
+
I0606 10:50:01.876816 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1483_pred.
|
| 308 |
+
I0606 10:50:01.960468 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 309 |
+
I0606 10:50:01.961593 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1426.
|
| 310 |
+
I0606 10:50:02.017819 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 311 |
+
I0606 10:50:02.018884 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1426_pred.
|
| 312 |
+
I0606 10:50:02.096171 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 313 |
+
I0606 10:50:02.097533 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451.
|
| 314 |
+
I0606 10:50:02.152100 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 315 |
+
I0606 10:50:02.153071 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451_pred.
|
| 316 |
+
I0606 10:50:02.232610 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 317 |
+
I0606 10:50:02.233661 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1344.
|
| 318 |
+
I0606 10:50:02.285053 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 319 |
+
I0606 10:50:02.285955 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1344_pred.
|
| 320 |
+
I0606 10:50:02.370647 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 321 |
+
I0606 10:50:02.371682 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0806.
|
| 322 |
+
I0606 10:50:02.428305 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 323 |
+
I0606 10:50:02.429317 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0806_pred.
|
| 324 |
+
I0606 10:50:02.501137 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 325 |
+
I0606 10:50:02.502440 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0582.
|
| 326 |
+
I0606 10:50:02.559240 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 327 |
+
I0606 10:50:02.560170 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0582_pred.
|
| 328 |
+
I0606 10:50:02.651891 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 329 |
+
I0606 10:50:02.655480 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0142.
|
| 330 |
+
I0606 10:50:02.716629 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 331 |
+
I0606 10:50:02.717769 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0142_pred.
|
| 332 |
+
I0606 10:50:02.789163 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 333 |
+
I0606 10:50:02.790483 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451_pred.
|
| 334 |
+
I0606 10:50:02.863391 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 335 |
+
I0606 10:50:02.864434 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0582_pred.
|
| 336 |
+
I0606 10:50:02.952141 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 337 |
+
I0606 10:50:02.953196 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1232.
|
| 338 |
+
I0606 10:50:03.006418 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 339 |
+
I0606 10:50:03.007707 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1232_pred.
|
| 340 |
+
I0606 10:50:03.084401 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 341 |
+
I0606 10:50:03.085415 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1085.
|
| 342 |
+
I0606 10:50:03.141296 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 343 |
+
I0606 10:50:03.142294 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1085_pred.
|
| 344 |
+
I0606 10:50:03.209192 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 345 |
+
I0606 10:50:03.210654 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0142_pred.
|
| 346 |
+
I0606 10:50:03.290408 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 347 |
+
I0606 10:50:03.291753 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1074.
|
| 348 |
+
I0606 10:50:03.354182 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 349 |
+
I0606 10:50:03.355217 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1074_pred.
|
| 350 |
+
I0606 10:50:03.436698 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 351 |
+
I0606 10:50:03.437797 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1456.
|
| 352 |
+
I0606 10:50:03.485870 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 353 |
+
I0606 10:50:03.486934 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1456_pred.
|
| 354 |
+
I0606 10:50:03.563019 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 355 |
+
I0606 10:50:03.564095 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0208_pred.
|
| 356 |
+
I0606 10:50:03.656216 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 357 |
+
I0606 10:50:03.657271 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0470.
|
| 358 |
+
I0606 10:50:03.723486 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 359 |
+
I0606 10:50:03.724655 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0470_pred.
|
| 360 |
+
I0606 10:50:03.804651 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 361 |
+
I0606 10:50:03.806172 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0347.
|
| 362 |
+
I0606 10:50:03.861241 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 363 |
+
I0606 10:50:03.862230 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0347_pred.
|
| 364 |
+
I0606 10:50:03.943164 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 365 |
+
I0606 10:50:03.944225 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451_pred.
|
| 366 |
+
I0606 10:50:04.022770 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 367 |
+
I0606 10:50:04.023969 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0417.
|
| 368 |
+
I0606 10:50:04.080241 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 369 |
+
I0606 10:50:04.081303 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0417_pred.
|
| 370 |
+
I0606 10:50:04.172245 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 371 |
+
I0606 10:50:04.173306 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1086_pred.
|
| 372 |
+
I0606 10:50:04.262993 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 373 |
+
I0606 10:50:04.264019 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0668.
|
| 374 |
+
I0606 10:50:04.333579 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 375 |
+
I0606 10:50:04.334654 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0668_pred.
|
| 376 |
+
I0606 10:50:04.418610 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 377 |
+
I0606 10:50:04.420113 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0090.
|
| 378 |
+
I0606 10:50:04.474496 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 379 |
+
I0606 10:50:04.475527 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0090_pred.
|
| 380 |
+
I0606 10:50:04.560549 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 381 |
+
I0606 10:50:04.561633 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1074_pred.
|
| 382 |
+
I0606 10:50:04.640963 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 383 |
+
I0606 10:50:04.642042 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1276.
|
| 384 |
+
I0606 10:50:04.694885 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 385 |
+
I0606 10:50:04.696139 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1276_pred.
|
| 386 |
+
I0606 10:50:04.775938 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 387 |
+
I0606 10:50:04.776987 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0619.
|
| 388 |
+
I0606 10:50:04.840255 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 389 |
+
I0606 10:50:04.841348 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0619_pred.
|
| 390 |
+
I0606 10:50:04.920950 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 391 |
+
I0606 10:50:04.922106 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1483_pred.
|
| 392 |
+
Epoch 0: sampling: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [01:13<00:00, 36.88s/it]
|
| 393 |
+
I0606 10:50:37.294374 140545645860672 train_omnigen.py:1279] No-grad rollout sampling finished in 73.8s; waiting for radiomics rewards
|
| 394 |
+
Waiting for rewards: 50%|██████████████████████████████████████████████████ | 1/2 [00:00<00:00, 2.17it/s]I0606 10:50:37.768049 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 395 |
+
I0606 10:50:37.769223 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1086_pred.
|
| 396 |
+
I0606 10:50:37.847552 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 397 |
+
I0606 10:50:37.848621 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0939_pred.
|
| 398 |
+
I0606 10:50:37.918017 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 399 |
+
I0606 10:50:37.919152 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0208_pred.
|
| 400 |
+
I0606 10:50:37.993597 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 401 |
+
I0606 10:50:37.995101 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0481_pred.
|
| 402 |
+
I0606 10:50:38.067920 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 403 |
+
I0606 10:50:38.068944 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0630_pred.
|
| 404 |
+
I0606 10:50:38.138463 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 405 |
+
I0606 10:50:38.139523 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0683_pred.
|
| 406 |
+
I0606 10:50:38.206979 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 407 |
+
I0606 10:50:38.208494 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1483_pred.
|
| 408 |
+
I0606 10:50:38.271194 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 409 |
+
I0606 10:50:38.272161 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1426_pred.
|
| 410 |
+
I0606 10:50:38.345165 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 411 |
+
I0606 10:50:38.346207 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451_pred.
|
| 412 |
+
I0606 10:50:38.415240 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 413 |
+
I0606 10:50:38.416396 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1344_pred.
|
| 414 |
+
I0606 10:50:38.484250 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 415 |
+
I0606 10:50:38.485252 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0806_pred.
|
| 416 |
+
I0606 10:50:38.557311 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 417 |
+
I0606 10:50:38.558356 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0582_pred.
|
| 418 |
+
I0606 10:50:38.635859 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 419 |
+
I0606 10:50:38.636907 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0142_pred.
|
| 420 |
+
I0606 10:50:38.707682 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 421 |
+
I0606 10:50:38.709202 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451_pred.
|
| 422 |
+
I0606 10:50:38.771737 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 423 |
+
I0606 10:50:38.772685 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0582_pred.
|
| 424 |
+
I0606 10:50:38.853368 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 425 |
+
I0606 10:50:38.854418 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1232_pred.
|
| 426 |
+
I0606 10:50:38.924938 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 427 |
+
I0606 10:50:38.926020 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1085_pred.
|
| 428 |
+
I0606 10:50:38.987672 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 429 |
+
I0606 10:50:38.988989 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0142_pred.
|
| 430 |
+
I0606 10:50:39.059872 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 431 |
+
I0606 10:50:39.060913 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1074_pred.
|
| 432 |
+
I0606 10:50:39.130846 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 433 |
+
I0606 10:50:39.131916 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1456_pred.
|
| 434 |
+
I0606 10:50:39.197305 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 435 |
+
I0606 10:50:39.198986 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0208_pred.
|
| 436 |
+
I0606 10:50:39.272849 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 437 |
+
I0606 10:50:39.273896 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0470_pred.
|
| 438 |
+
I0606 10:50:39.352105 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 439 |
+
I0606 10:50:39.353142 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0347_pred.
|
| 440 |
+
I0606 10:50:39.427494 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 441 |
+
I0606 10:50:39.428550 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451_pred.
|
| 442 |
+
I0606 10:50:39.493806 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 443 |
+
I0606 10:50:39.495317 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0417_pred.
|
| 444 |
+
I0606 10:50:39.568905 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 445 |
+
I0606 10:50:39.569937 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1086_pred.
|
| 446 |
+
I0606 10:50:39.645720 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 447 |
+
I0606 10:50:39.646769 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0668_pred.
|
| 448 |
+
I0606 10:50:39.722388 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 449 |
+
I0606 10:50:39.723459 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0090_pred.
|
| 450 |
+
I0606 10:50:39.793580 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 451 |
+
I0606 10:50:39.795033 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1074_pred.
|
| 452 |
+
I0606 10:50:39.864843 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 453 |
+
I0606 10:50:39.865874 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1276_pred.
|
| 454 |
+
I0606 10:50:39.937602 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 455 |
+
I0606 10:50:39.938608 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0619_pred.
|
| 456 |
+
I0606 10:50:40.015681 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 457 |
+
I0606 10:50:40.016846 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1483_pred.
|
| 458 |
+
Waiting for rewards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:02<00:00, 1.39s/it]
|
| 459 |
+
I0606 10:50:40.071026 140545645860672 train_omnigen.py:1295] Radiomics reward wait finished in 2.8s
|
| 460 |
+
I0606 10:53:37.966390 140545645860672 train_omnigen.py:1434] Epoch 0 reward summary: avg=-0.8839, radiomics=-0.8839, radiomics_distance=0.8839, radiomics_missing=0.0000, radiomics_roi_pixels=30609.8516, zero_std_ratio=0.0000, reward_std_mean=0.1873, group_size=4.34, trained_prompt_num=59
|
| 461 |
+
I0606 10:53:37.970433 140545645860672 train_omnigen.py:1455] Starting RL training updates
|
| 462 |
+
Epoch 0.0: training: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [04:49<00:00, 144.72s/it]
|
| 463 |
+
I0606 10:58:27.420725 140545645860672 train_omnigen.py:1574] RL update phase finished in 289.5s
|
| 464 |
+
I0606 10:58:27.423767 140545645860672 train_omnigen.py:1179] Starting no-grad rollout sampling for epoch 1
|
| 465 |
+
Epoch 1: sampling: 50%|███████████████████████████████████████████████████ | 1/2 [00:36<00:36, 36.61s/it]I0606 10:59:04.513085 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 466 |
+
I0606 10:59:04.514278 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1086_pred.
|
| 467 |
+
I0606 10:59:04.584459 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 468 |
+
I0606 10:59:04.585470 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0939_pred.
|
| 469 |
+
I0606 10:59:04.653365 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 470 |
+
I0606 10:59:04.654335 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0208_pred.
|
| 471 |
+
I0606 10:59:04.731165 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 472 |
+
I0606 10:59:04.732172 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0481_pred.
|
| 473 |
+
I0606 10:59:04.810816 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 474 |
+
I0606 10:59:04.811854 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0630_pred.
|
| 475 |
+
I0606 10:59:04.882081 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 476 |
+
I0606 10:59:04.883103 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0683_pred.
|
| 477 |
+
I0606 10:59:04.952876 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 478 |
+
I0606 10:59:04.953960 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1483_pred.
|
| 479 |
+
I0606 10:59:05.026894 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 480 |
+
I0606 10:59:05.028030 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1426_pred.
|
| 481 |
+
I0606 10:59:05.105101 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 482 |
+
I0606 10:59:05.106302 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451_pred.
|
| 483 |
+
I0606 10:59:05.175969 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 484 |
+
I0606 10:59:05.177019 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1344_pred.
|
| 485 |
+
I0606 10:59:05.245570 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 486 |
+
I0606 10:59:05.246539 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0806_pred.
|
| 487 |
+
I0606 10:59:05.321919 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 488 |
+
I0606 10:59:05.323064 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0582_pred.
|
| 489 |
+
I0606 10:59:05.398912 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 490 |
+
I0606 10:59:05.400523 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0142_pred.
|
| 491 |
+
I0606 10:59:05.469727 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 492 |
+
I0606 10:59:05.470760 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451_pred.
|
| 493 |
+
I0606 10:59:05.545255 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 494 |
+
I0606 10:59:05.546314 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0582_pred.
|
| 495 |
+
I0606 10:59:05.631663 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 496 |
+
I0606 10:59:05.632746 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1232_pred.
|
| 497 |
+
I0606 10:59:05.704937 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 498 |
+
I0606 10:59:05.706520 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1085_pred.
|
| 499 |
+
I0606 10:59:05.775474 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 500 |
+
I0606 10:59:05.776485 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0142_pred.
|
| 501 |
+
I0606 10:59:05.861274 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 502 |
+
I0606 10:59:05.862365 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1074_pred.
|
| 503 |
+
I0606 10:59:05.938323 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 504 |
+
I0606 10:59:05.939375 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1456_pred.
|
| 505 |
+
I0606 10:59:06.012083 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 506 |
+
I0606 10:59:06.013256 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0208_pred.
|
| 507 |
+
I0606 10:59:06.092206 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 508 |
+
I0606 10:59:06.093825 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0470_pred.
|
| 509 |
+
I0606 10:59:06.171501 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 510 |
+
I0606 10:59:06.172549 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0347_pred.
|
| 511 |
+
I0606 10:59:06.254780 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 512 |
+
I0606 10:59:06.255844 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1451_pred.
|
| 513 |
+
I0606 10:59:06.331206 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 514 |
+
I0606 10:59:06.332278 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0417_pred.
|
| 515 |
+
I0606 10:59:06.411033 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 516 |
+
I0606 10:59:06.412245 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1086_pred.
|
| 517 |
+
I0606 10:59:06.482770 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 518 |
+
I0606 10:59:06.483769 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0668_pred.
|
| 519 |
+
I0606 10:59:06.564175 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 520 |
+
I0606 10:59:06.565235 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0090_pred.
|
| 521 |
+
I0606 10:59:06.648903 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 522 |
+
I0606 10:59:06.649960 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1074_pred.
|
| 523 |
+
I0606 10:59:06.723650 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 524 |
+
I0606 10:59:06.724747 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1276_pred.
|
| 525 |
+
I0606 10:59:06.795164 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 526 |
+
I0606 10:59:06.797748 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0619_pred.
|
| 527 |
+
I0606 10:59:06.868018 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 528 |
+
I0606 10:59:06.869048 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1483_pred.
|
| 529 |
+
Epoch 1: sampling: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [01:13<00:00, 36.60s/it]
|
| 530 |
+
I0606 10:59:40.615388 140545645860672 train_omnigen.py:1279] No-grad rollout sampling finished in 73.2s; waiting for radiomics rewards
|
| 531 |
+
Waiting for rewards: 50%|██████████████████████████████████████████████████ | 1/2 [00:00<00:00, 2.17it/s]I0606 10:59:41.096873 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 532 |
+
I0606 10:59:41.098453 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0022.
|
| 533 |
+
I0606 10:59:41.155526 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 534 |
+
I0606 10:59:41.156518 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0022_pred.
|
| 535 |
+
I0606 10:59:41.241573 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 536 |
+
I0606 10:59:41.242621 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0653.
|
| 537 |
+
I0606 10:59:41.292186 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 538 |
+
I0606 10:59:41.293447 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0653_pred.
|
| 539 |
+
I0606 10:59:41.371578 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 540 |
+
I0606 10:59:41.372587 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0018.
|
| 541 |
+
I0606 10:59:41.434468 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 542 |
+
I0606 10:59:41.435502 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0018_pred.
|
| 543 |
+
I0606 10:59:41.518553 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 544 |
+
I0606 10:59:41.519675 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1124.
|
| 545 |
+
I0606 10:59:41.571053 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 546 |
+
I0606 10:59:41.571942 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1124_pred.
|
| 547 |
+
I0606 10:59:41.648346 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 548 |
+
I0606 10:59:41.649348 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0290.
|
| 549 |
+
I0606 10:59:41.700508 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 550 |
+
I0606 10:59:41.701633 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0290_pred.
|
| 551 |
+
I0606 10:59:41.772088 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 552 |
+
I0606 10:59:41.773100 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0290_pred.
|
| 553 |
+
I0606 10:59:41.851846 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 554 |
+
I0606 10:59:41.852887 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0507.
|
| 555 |
+
I0606 10:59:41.897215 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 556 |
+
I0606 10:59:41.898269 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0507_pred.
|
| 557 |
+
I0606 10:59:41.967801 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 558 |
+
I0606 10:59:41.968794 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1086.
|
| 559 |
+
I0606 10:59:42.016909 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 560 |
+
I0606 10:59:42.017944 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1086_pred.
|
| 561 |
+
I0606 10:59:42.088827 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 562 |
+
I0606 10:59:42.089825 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0872.
|
| 563 |
+
I0606 10:59:42.143742 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 564 |
+
I0606 10:59:42.144762 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0872_pred.
|
| 565 |
+
I0606 10:59:42.214497 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 566 |
+
I0606 10:59:42.215576 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0022_pred.
|
| 567 |
+
I0606 10:59:42.294539 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 568 |
+
I0606 10:59:42.296320 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0927.
|
| 569 |
+
I0606 10:59:42.343006 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 570 |
+
I0606 10:59:42.343978 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0927_pred.
|
| 571 |
+
I0606 10:59:42.415258 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 572 |
+
I0606 10:59:42.416357 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0884.
|
| 573 |
+
I0606 10:59:42.463708 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 574 |
+
I0606 10:59:42.464601 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0884_pred.
|
| 575 |
+
I0606 10:59:42.535948 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 576 |
+
I0606 10:59:42.536956 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0996.
|
| 577 |
+
I0606 10:59:42.588476 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 578 |
+
I0606 10:59:42.589402 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0996_pred.
|
| 579 |
+
I0606 10:59:42.665207 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 580 |
+
I0606 10:59:42.666208 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0724.
|
| 581 |
+
I0606 10:59:42.720703 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 582 |
+
I0606 10:59:42.721736 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0724_pred.
|
| 583 |
+
I0606 10:59:42.795597 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 584 |
+
I0606 10:59:42.796690 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1318.
|
| 585 |
+
I0606 10:59:42.845396 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 586 |
+
I0606 10:59:42.846383 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1318_pred.
|
| 587 |
+
I0606 10:59:42.918637 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 588 |
+
I0606 10:59:42.919690 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0032.
|
| 589 |
+
I0606 10:59:42.969358 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 590 |
+
I0606 10:59:42.970231 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0032_pred.
|
| 591 |
+
I0606 10:59:43.037615 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 592 |
+
I0606 10:59:43.038626 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0053.
|
| 593 |
+
I0606 10:59:43.093817 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 594 |
+
I0606 10:59:43.094833 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0053_pred.
|
| 595 |
+
I0606 10:59:43.177621 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 596 |
+
I0606 10:59:43.178628 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0952.
|
| 597 |
+
I0606 10:59:43.225898 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 598 |
+
I0606 10:59:43.226922 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0952_pred.
|
| 599 |
+
I0606 10:59:43.297687 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 600 |
+
I0606 10:59:43.298780 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1167.
|
| 601 |
+
I0606 10:59:43.348173 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 602 |
+
I0606 10:59:43.349128 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1167_pred.
|
| 603 |
+
I0606 10:59:43.412408 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 604 |
+
I0606 10:59:43.413509 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0884_pred.
|
| 605 |
+
I0606 10:59:43.483965 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 606 |
+
I0606 10:59:43.484946 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0268.
|
| 607 |
+
I0606 10:59:43.535511 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 608 |
+
I0606 10:59:43.536520 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0268_pred.
|
| 609 |
+
I0606 10:59:43.612015 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 610 |
+
I0606 10:59:43.613129 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0961.
|
| 611 |
+
I0606 10:59:43.663554 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 612 |
+
I0606 10:59:43.664459 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0961_pred.
|
| 613 |
+
I0606 10:59:43.739377 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 614 |
+
I0606 10:59:43.740431 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1345.
|
| 615 |
+
I0606 10:59:43.785642 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 616 |
+
I0606 10:59:43.786509 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1345_pred.
|
| 617 |
+
I0606 10:59:43.855051 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 618 |
+
I0606 10:59:43.856056 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1258.
|
| 619 |
+
I0606 10:59:43.902014 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 620 |
+
I0606 10:59:43.903113 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1258_pred.
|
| 621 |
+
I0606 10:59:43.970291 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 622 |
+
I0606 10:59:43.971261 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0617.
|
| 623 |
+
I0606 10:59:44.024327 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 624 |
+
I0606 10:59:44.025353 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0617_pred.
|
| 625 |
+
I0606 10:59:44.094654 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 626 |
+
I0606 10:59:44.095799 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0290_pred.
|
| 627 |
+
I0606 10:59:44.170575 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 628 |
+
I0606 10:59:44.171583 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0452.
|
| 629 |
+
I0606 10:59:44.228057 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 630 |
+
I0606 10:59:44.229090 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0452_pred.
|
| 631 |
+
I0606 10:59:44.299900 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 632 |
+
I0606 10:59:44.301124 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1318_pred.
|
| 633 |
+
I0606 10:59:44.369925 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 634 |
+
I0606 10:59:44.370959 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0452_pred.
|
| 635 |
+
I0606 10:59:44.448371 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 636 |
+
I0606 10:59:44.449413 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0453.
|
| 637 |
+
I0606 10:59:44.499730 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 638 |
+
I0606 10:59:44.501757 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0453_pred.
|
| 639 |
+
I0606 10:59:44.572497 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 640 |
+
I0606 10:59:44.573484 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0509.
|
| 641 |
+
I0606 10:59:44.624491 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 642 |
+
I0606 10:59:44.625510 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 0509_pred.
|
| 643 |
+
I0606 10:59:44.690991 140528929109696 extract_features_and_images.py:361] Jobs are processed sequentially.
|
| 644 |
+
I0606 10:59:44.692007 140528929109696 standardWorkflow.py:82] Initialising feature computation using dx images for 1167_pred.
|
| 645 |
+
Waiting for rewards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.07s/it]
|
| 646 |
+
I0606 10:59:44.748656 140545645860672 train_omnigen.py:1295] Radiomics reward wait finished in 4.1s
|
| 647 |
+
I0606 10:59:46.570990 140545645860672 train_omnigen.py:1434] Epoch 1 reward summary: avg=-0.8679, radiomics=-0.8679, radiomics_distance=0.8679, radiomics_missing=0.0000, radiomics_roi_pixels=30475.3633, zero_std_ratio=0.2857, reward_std_mean=0.1344, group_size=2.15, trained_prompt_num=119
|
| 648 |
+
I0606 10:59:46.571547 140545645860672 train_omnigen.py:1455] Starting RL training updates
|
| 649 |
+
Epoch 1.0: training: 0%| | 0/2 [01:40<?, ?it/s]
|
| 650 |
+
Traceback (most recent call last):
|
| 651 |
+
File "/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/scripts/train_omnigen.py", line 1581, in <module>
|
| 652 |
+
app.run(main)
|
| 653 |
+
File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/absl/app.py", line 367, in run
|
| 654 |
+
_run_main(main, args)
|
| 655 |
+
File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/absl/app.py", line 312, in _run_main
|
| 656 |
+
sys.exit(main(argv))
|
| 657 |
+
File "/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/scripts/train_omnigen.py", line 1549, in main
|
| 658 |
+
accelerator.backward(loss)
|
| 659 |
+
File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/accelerate/accelerator.py", line 2329, in backward
|
| 660 |
+
loss.backward(**kwargs)
|
| 661 |
+
File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/torch/_tensor.py", line 626, in backward
|
| 662 |
+
torch.autograd.backward(
|
| 663 |
+
File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 664 |
+
_engine_run_backward(
|
| 665 |
+
File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/torch/autograd/graph.py", line 823, in _engine_run_backward
|
| 666 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 667 |
+
KeyboardInterrupt
|
| 668 |
+
[rank0]: Traceback (most recent call last):
|
| 669 |
+
[rank0]: File "/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/scripts/train_omnigen.py", line 1581, in <module>
|
| 670 |
+
[rank0]: app.run(main)
|
| 671 |
+
[rank0]: File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/absl/app.py", line 367, in run
|
| 672 |
+
[rank0]: _run_main(main, args)
|
| 673 |
+
[rank0]: File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/absl/app.py", line 312, in _run_main
|
| 674 |
+
[rank0]: sys.exit(main(argv))
|
| 675 |
+
[rank0]: File "/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/scripts/train_omnigen.py", line 1549, in main
|
| 676 |
+
[rank0]: accelerator.backward(loss)
|
| 677 |
+
[rank0]: File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/accelerate/accelerator.py", line 2329, in backward
|
| 678 |
+
[rank0]: loss.backward(**kwargs)
|
| 679 |
+
[rank0]: File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/torch/_tensor.py", line 626, in backward
|
| 680 |
+
[rank0]: torch.autograd.backward(
|
| 681 |
+
[rank0]: File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 682 |
+
[rank0]: _engine_run_backward(
|
| 683 |
+
[rank0]: File "/root/miniconda3/envs/flow-grpo/lib/python3.10/site-packages/torch/autograd/graph.py", line 823, in _engine_run_backward
|
| 684 |
+
[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 685 |
+
[rank0]: KeyboardInterrupt
|
run-20260606_101540-71wbmn3f/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.4.241-1-tlinux4-0017.7-x86_64-with-glibc2.38",
|
| 3 |
+
"python": "3.10.20",
|
| 4 |
+
"startedAt": "2026-06-06T02:15:40.738788Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"config/grpo.py:general_radiomics_omnigen_4gpu_kl",
|
| 8 |
+
"--config.save_freq=5"
|
| 9 |
+
],
|
| 10 |
+
"program": "/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/scripts/train_omnigen.py",
|
| 11 |
+
"codePath": "scripts/train_omnigen.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/zhui711/flow_grpo_cxr_servera55.git",
|
| 14 |
+
"commit": "5b537cea6dba56f0f6308ae8ba43c6f95afaa1bd"
|
| 15 |
+
},
|
| 16 |
+
"email": "1341610137@qq.com",
|
| 17 |
+
"root": "/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr",
|
| 18 |
+
"host": "TENCENT64.site",
|
| 19 |
+
"username": "root",
|
| 20 |
+
"executable": "/root/miniconda3/envs/flow-grpo/bin/python3",
|
| 21 |
+
"codePathLocal": "scripts/train_omnigen.py",
|
| 22 |
+
"cpu_count": 192,
|
| 23 |
+
"cpu_count_logical": 384,
|
| 24 |
+
"gpu": "NVIDIA H20",
|
| 25 |
+
"gpu_count": 8,
|
| 26 |
+
"disk": {
|
| 27 |
+
"/": {
|
| 28 |
+
"total": "12800361496576",
|
| 29 |
+
"used": "4526144532480"
|
| 30 |
+
}
|
| 31 |
+
},
|
| 32 |
+
"memory": {
|
| 33 |
+
"total": "2432295661568"
|
| 34 |
+
},
|
| 35 |
+
"cpu": {
|
| 36 |
+
"count": 192,
|
| 37 |
+
"countLogical": 384
|
| 38 |
+
},
|
| 39 |
+
"gpu_nvidia": [
|
| 40 |
+
{
|
| 41 |
+
"name": "NVIDIA H20",
|
| 42 |
+
"memoryTotal": "102625181696",
|
| 43 |
+
"cudaCores": 9984,
|
| 44 |
+
"architecture": "Hopper"
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"name": "NVIDIA H20",
|
| 48 |
+
"memoryTotal": "102625181696",
|
| 49 |
+
"cudaCores": 9984,
|
| 50 |
+
"architecture": "Hopper"
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"name": "NVIDIA H20",
|
| 54 |
+
"memoryTotal": "102625181696",
|
| 55 |
+
"cudaCores": 9984,
|
| 56 |
+
"architecture": "Hopper"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"name": "NVIDIA H20",
|
| 60 |
+
"memoryTotal": "102625181696",
|
| 61 |
+
"cudaCores": 9984,
|
| 62 |
+
"architecture": "Hopper"
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"name": "NVIDIA H20",
|
| 66 |
+
"memoryTotal": "102625181696",
|
| 67 |
+
"cudaCores": 9984,
|
| 68 |
+
"architecture": "Hopper"
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"name": "NVIDIA H20",
|
| 72 |
+
"memoryTotal": "102625181696",
|
| 73 |
+
"cudaCores": 9984,
|
| 74 |
+
"architecture": "Hopper"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"name": "NVIDIA H20",
|
| 78 |
+
"memoryTotal": "102625181696",
|
| 79 |
+
"cudaCores": 9984,
|
| 80 |
+
"architecture": "Hopper"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"name": "NVIDIA H20",
|
| 84 |
+
"memoryTotal": "102625181696",
|
| 85 |
+
"cudaCores": 9984,
|
| 86 |
+
"architecture": "Hopper"
|
| 87 |
+
}
|
| 88 |
+
],
|
| 89 |
+
"cudaVersion": "12.8"
|
| 90 |
+
}
|
run-20260606_101540-71wbmn3f/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"clipfrac_lt_one":0,"group_size":2.1512605042016806,"reward_radiomics_roi_pixels":30475.36328125,"kl_loss":0.13080264627933502,"reward_radiomics_distance":0.8679078817367554,"epoch":1,"approx_kl":1.6419642034914972e-10,"eval_reward_radiomics":-0.7145048379898071,"_step":2,"_wandb":{"runtime":2745},"trained_prompt_num":119,"reward/radiomics_missing_score_rate":0,"reward_ori_avg":-0.8679078817367554,"eval_images":{"captions":["prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.0247 rad, sin(d_azimuth)=0.0269, cos(d_azimuth)=0.9996.\nradiomics: -0.27 | radiomics_distance: 0.27 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30447.00 | avg: -0.27\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.0515 rad, sin(d_azimuth)=-0.0045, cos(d_azimuth)=1.0000.\nradiomics: -0.37 | radiomics_distance: 0.37 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30732.00 | avg: -0.37\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.0502 rad, sin(d_azimuth)=-0.0385, cos(d_azimuth)=0.9993.\nradiomics: -0.13 | radiomics_distance: 0.13 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30057.00 | avg: -0.13\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.0127 rad, sin(d_azimuth)=0.0719, cos(d_azimuth)=0.9974.\nradiomics: -0.41 | radiomics_distance: 0.41 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30781.00 | avg: -0.41\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.0438 rad, sin(d_azimuth)=-0.0689, cos(d_azimuth)=0.9976.\nradiomics: -0.24 | radiomics_distance: 0.24 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30550.00 | avg: -0.24\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.0864 rad, sin(d_azimuth)=0.0233, cos(d_azimuth)=0.9997.\nradiomics: -0.32 | radiomics_distance: 0.32 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30294.00 | avg: -0.32\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.0858 rad, sin(d_azimuth)=0.0446, cos(d_azimuth)=0.9990.\nradiomics: -0.25 | radiomics_distance: 0.25 | radiomics_missing: 0.00 | radiomics_roi_pixels: 31368.00 | avg: -0.25\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.0354 rad, sin(d_azimuth)=-0.0970, cos(d_azimuth)=0.9953.\nradiomics: -0.34 | radiomics_distance: 0.34 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30384.00 | avg: -0.34\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.0418 rad, sin(d_azimuth)=0.1012, cos(d_azimuth)=0.9949.\nradiomics: -0.92 | radiomics_distance: 0.92 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30594.00 | avg: -0.92\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.1046 rad, sin(d_azimuth)=-0.0491, cos(d_azimuth)=0.9988.\nradiomics: -0.11 | radiomics_distance: 0.11 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30896.00 | avg: -0.11\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.1156 rad, sin(d_azimuth)=-0.0364, cos(d_azimuth)=0.9993.\nradiomics: -0.20 | radiomics_distance: 0.20 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30044.00 | avg: -0.20\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.0633 rad, sin(d_azimuth)=0.1095, cos(d_azimuth)=0.9940.\nradiomics: -0.51 | radiomics_distance: 0.51 | radiomics_missing: 0.00 | radiomics_roi_pixels: 31466.00 | avg: -0.51\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.0282 rad, sin(d_azimuth)=-0.1284, cos(d_azimuth)=0.9917.\nradiomics: -0.60 | radiomics_distance: 0.60 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30494.00 | avg: -0.60\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.1118 rad, sin(d_azimuth)=0.0789, cos(d_azimuth)=0.9969.\nradiomics: -0.47 | radiomics_distance: 0.47 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30328.00 | avg: -0.47\neval output","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.1404 rad, sin(d_azimuth)=0.0183, cos(d_azimuth)=0.9998.\nradiomics: -0.18 | radiomics_distance: 0.18 | radiomics_missing: 0.00 | radiomics_roi_pixels: 31427.00 | avg: -0.18\neval output"],"_type":"images/separated","width":524,"height":280,"format":"png","count":15,"filenames":["media/images/eval_images_0_66feaebf5350fc850b80.png","media/images/eval_images_0_243c70f367b210f65092.png","media/images/eval_images_0_ac50aa2464a6912a75bf.png","media/images/eval_images_0_d81fc34b4e3bc9379445.png","media/images/eval_images_0_43cdd7a37f85c6a13c2e.png","media/images/eval_images_0_ac7ce769f360461ad2ea.png","media/images/eval_images_0_d4e7c1934d99188c6708.png","media/images/eval_images_0_d261a691537a4ec6a12b.png","media/images/eval_images_0_3b28a4e69bd3270b76ad.png","media/images/eval_images_0_fd4bdd1ff66d8ba33988.png","media/images/eval_images_0_6f4aa734b7c88a6d9df9.png","media/images/eval_images_0_0cb1045c40045005886e.png","media/images/eval_images_0_78450a1ff8752a736686.png","media/images/eval_images_0_1f7b70d4340f1682d116.png","media/images/eval_images_0_97c186f4c58d304f4e76.png"]},"config/use_joint_mask":0,"zero_std_ratio":0.2857142857142857,"train_rollout_images":{"height":280,"format":"png","count":8,"filenames":["media/images/train_rollout_images_0_88a3182e8d6a3b61e4c3.png","media/images/train_rollout_images_0_6c599567e8776dc5cbf9.png","media/images/train_rollout_images_0_518103ac97c84ab1036d.png","media/images/train_rollout_images_0_f88f9c1b09e0d91499c1.png","media/images/train_rollout_images_0_feb9cce0d402d0bdf356.png","media/images/train_rollout_images_0_2296d182e5a0026c5b49.png","media/images/train_rollout_images_0_875c04962a4a6cb88d52.png","media/images/train_rollout_images_0_db13edf7b89f80a636f2.png"],"captions":["prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=1.0800 rad, sin(d_azimuth)=-0.8110, cos(d_azimuth)=0.5851.\nradiomics: -1.63 | radiomics_distance: 1.63 | radiomics_missing: 0.00 | radiomics_roi_pixels: 30218.00 | avg: -1.63\nstochastic rollout (noise_level=0.01)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.9299 rad, sin(d_azimuth)=0.7805, cos(d_azimuth)=0.6251.\nradiomics: -1.12 | radiomics_distance: 1.12 | radiomics_missing: 0.00 | radiomics_roi_pixels: 23026.00 | avg: -1.12\nstochastic rollout (noise_level=0.01)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.1610 rad, sin(d_azimuth)=0.4885, cos(d_azimuth)=0.8726.\nradiomics: -1.26 | radiomics_distance: 1.26 | radiomics_missing: 0.00 | radiomics_roi_pixels: 39361.00 | avg: -1.26\nstochastic rollout (noise_level=0.01)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.8116 rad, sin(d_azimuth)=0.1625, cos(d_azimuth)=0.9867.\nradiomics: -1.19 | radiomics_distance: 1.19 | radiomics_missing: 0.00 | radiomics_roi_pixels: 36259.00 | avg: -1.19\nstochastic rollout (noise_level=0.01)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.6727 rad, sin(d_azimuth)=0.6711, cos(d_azimuth)=0.7414.\nradiomics: -0.44 | radiomics_distance: 0.44 | radiomics_missing: 0.00 | radiomics_roi_pixels: 27551.00 | avg: -0.44\nstochastic rollout (noise_level=0.01)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.5985 rad, sin(d_azimuth)=-0.7521, cos(d_azimuth)=0.6591.\nradiomics: -0.68 | radiomics_distance: 0.68 | radiomics_missing: 0.00 | radiomics_roi_pixels: 26717.00 | avg: -0.68\nstochastic rollout (noise_level=0.01)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.2790 rad, sin(d_azimuth)=0.9999, cos(d_azimuth)=0.0114.\nradiomics: -1.11 | radiomics_distance: 1.11 | radiomics_missing: 0.00 | radiomics_roi_pixels: 25754.00 | avg: -1.11\nstochastic rollout (noise_level=0.01)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=1.1505 rad, sin(d_azimuth)=0.9928, cos(d_azimuth)=0.1201.\nradiomics: -0.76 | radiomics_distance: 0.76 | radiomics_missing: 0.00 | radiomics_roi_pixels: 31138.00 | avg: -0.76\nstochastic rollout (noise_level=0.01)"],"_type":"images/separated","width":524},"policy_loss":-0.05693891644477844,"eval_reward_radiomics_roi_pixels":30951.01171875,"inner_epoch":0,"_runtime":2745.974064667,"clipfrac_gt_one":0,"eval_reward_avg":-0.7145048379898071,"eval_reward_radiomics_distance":0.7145048379898071,"images":{"filenames":["media/images/images_0_bc6a36f25e0e0cf9631f.png","media/images/images_0_90e8321e45af352448c5.png","media/images/images_0_2b1e366339eedfbce997.png","media/images/images_0_ecd25b0e9060ac0b15cf.png","media/images/images_0_605692edaa37b67dc68a.png","media/images/images_0_0a77a4658bd6cc6fca47.png","media/images/images_0_50d936f2e6f165ae7ff7.png","media/images/images_0_9e14dcb20935e442f24d.png"],"captions":["prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=1.0800 rad, sin(d_azimuth)=-0.8110, cos(d_azimuth)=0.5851.\ndeterministic preview (noise_level=0.0)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.9299 rad, sin(d_azimuth)=0.7805, cos(d_azimuth)=0.6251.\ndeterministic preview (noise_level=0.0)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.1610 rad, sin(d_azimuth)=0.4885, cos(d_azimuth)=0.8726.\ndeterministic preview (noise_level=0.0)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.8116 rad, sin(d_azimuth)=0.1625, cos(d_azimuth)=0.9867.\ndeterministic preview (noise_level=0.0)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.6727 rad, sin(d_azimuth)=0.6711, cos(d_azimuth)=0.7414.\ndeterministic preview (noise_level=0.0)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=0.5985 rad, sin(d_azimuth)=-0.7521, cos(d_azimuth)=0.6591.\ndeterministic preview (noise_level=0.0)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=-0.2790 rad, sin(d_azimuth)=0.9999, cos(d_azimuth)=0.0114.\ndeterministic preview (noise_level=0.0)","prompt: <img><|image_1|></img> Edit the view using delta pose: d_theta=1.1505 rad, sin(d_azimuth)=0.9928, cos(d_azimuth)=0.1201.\ndeterministic preview (noise_level=0.0)"],"_type":"images/separated","width":524,"height":280,"format":"png","count":8},"reward/roi_pixels_mean":30475.36328125,"eval_reward_radiomics_missing":0,"loss":-0.05628490447998047,"_timestamp":1.7807147865707934e+09,"reward_std_mean":0.1343677043914795,"reward_radiomics_missing":0,"reward_radiomics":-0.8679078817367554,"reward_avg":-0.8679078221321106,"clipfrac":0}
|
run-20260606_101540-71wbmn3f/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-06-06T10:15:40.740953441+08:00","level":"INFO","msg":"using version","core version":"0.18.7"}
|
| 2 |
+
{"time":"2026-06-06T10:15:40.741088088+08:00","level":"INFO","msg":"created symlink","path":"/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/wandb/run-20260606_101540-71wbmn3f/logs/debug-core.log"}
|
| 3 |
+
{"time":"2026-06-06T10:15:41.052440288+08:00","level":"INFO","msg":"created new stream","id":"71wbmn3f"}
|
| 4 |
+
{"time":"2026-06-06T10:15:41.052636214+08:00","level":"INFO","msg":"stream: started","id":"71wbmn3f"}
|
| 5 |
+
{"time":"2026-06-06T10:15:41.052722322+08:00","level":"INFO","msg":"sender: started","stream_id":"71wbmn3f"}
|
| 6 |
+
{"time":"2026-06-06T10:15:41.052665453+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"71wbmn3f"}
|
| 7 |
+
{"time":"2026-06-06T10:15:41.052664533+08:00","level":"INFO","msg":"handler: started","stream_id":"71wbmn3f"}
|
| 8 |
+
{"time":"2026-06-06T10:15:42.345026669+08:00","level":"INFO","msg":"Starting system monitor"}
|
| 9 |
+
{"time":"2026-06-06T11:01:26.712839127+08:00","level":"INFO","msg":"stream: closing","id":"71wbmn3f"}
|
| 10 |
+
{"time":"2026-06-06T11:01:26.712868376+08:00","level":"INFO","msg":"Stopping system monitor"}
|
| 11 |
+
{"time":"2026-06-06T11:01:26.713971633+08:00","level":"INFO","msg":"Stopped system monitor"}
|
| 12 |
+
{"time":"2026-06-06T11:01:31.908581259+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 13 |
+
{"time":"2026-06-06T11:01:32.581320012+08:00","level":"INFO","msg":"handler: closed","stream_id":"71wbmn3f"}
|
| 14 |
+
{"time":"2026-06-06T11:01:32.581341052+08:00","level":"INFO","msg":"sender: closed","stream_id":"71wbmn3f"}
|
| 15 |
+
{"time":"2026-06-06T11:01:32.581340262+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"71wbmn3f"}
|
| 16 |
+
{"time":"2026-06-06T11:01:32.582336811+08:00","level":"INFO","msg":"stream: closed","id":"71wbmn3f"}
|
run-20260606_101540-71wbmn3f/logs/debug.log
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-06-06 10:15:40,719 INFO MainThread:430688 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
|
| 2 |
+
2026-06-06 10:15:40,719 INFO MainThread:430688 [wandb_setup.py:_flush():79] Configure stats pid to 430688
|
| 3 |
+
2026-06-06 10:15:40,719 INFO MainThread:430688 [wandb_setup.py:_flush():79] Loading settings from /root/.config/wandb/settings
|
| 4 |
+
2026-06-06 10:15:40,719 INFO MainThread:430688 [wandb_setup.py:_flush():79] Loading settings from /apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/wandb/settings
|
| 5 |
+
2026-06-06 10:15:40,719 WARNING MainThread:430688 [wandb_setup.py:_flush():79] Unknown environment variable: WANDB_GROUP
|
| 6 |
+
2026-06-06 10:15:40,720 INFO MainThread:430688 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'project': 'flow_grpo_img_only_scratch30k', 'api_key': '***REDACTED***', 'resume': 'never', 'run_name': 'statcm_n001_bs32_steps200', 'mode': 'online', 'run_tags': ['img-only', 'scratch30k', 'manual', 'noise0.01', 'bs32']}
|
| 7 |
+
2026-06-06 10:15:40,720 INFO MainThread:430688 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
|
| 8 |
+
2026-06-06 10:15:40,720 INFO MainThread:430688 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'scripts/train_omnigen.py', 'program_abspath': '/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/scripts/train_omnigen.py', 'program': '/apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/scripts/train_omnigen.py'}
|
| 9 |
+
2026-06-06 10:15:40,720 INFO MainThread:430688 [wandb_setup.py:_flush():79] Applying login settings: {}
|
| 10 |
+
2026-06-06 10:15:40,721 INFO MainThread:430688 [wandb_init.py:_log_setup():533] Logging user logs to /apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/wandb/run-20260606_101540-71wbmn3f/logs/debug.log
|
| 11 |
+
2026-06-06 10:15:40,722 INFO MainThread:430688 [wandb_init.py:_log_setup():534] Logging internal logs to /apdcephfs_zwfy2/share_303944931/connorxian/Code/medUni/wt/zr/flow_grpo_cxr/wandb/run-20260606_101540-71wbmn3f/logs/debug-internal.log
|
| 12 |
+
2026-06-06 10:15:40,722 INFO MainThread:430688 [wandb_init.py:init():619] calling init triggers
|
| 13 |
+
2026-06-06 10:15:40,722 INFO MainThread:430688 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
|
| 14 |
+
config: {}
|
| 15 |
+
2026-06-06 10:15:40,722 INFO MainThread:430688 [wandb_init.py:init():669] starting backend
|
| 16 |
+
2026-06-06 10:15:40,722 INFO MainThread:430688 [wandb_init.py:init():673] sending inform_init request
|
| 17 |
+
2026-06-06 10:15:40,737 INFO MainThread:430688 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
| 18 |
+
2026-06-06 10:15:40,738 INFO MainThread:430688 [wandb_init.py:init():686] backend started and connected
|
| 19 |
+
2026-06-06 10:15:40,772 INFO MainThread:430688 [wandb_init.py:init():781] updated telemetry
|
| 20 |
+
2026-06-06 10:15:40,847 INFO MainThread:430688 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
|
| 21 |
+
2026-06-06 10:15:42,320 INFO MainThread:430688 [wandb_init.py:init():867] starting run threads in backend
|
| 22 |
+
2026-06-06 10:15:42,456 INFO MainThread:430688 [wandb_run.py:_console_start():2456] atexit reg
|
| 23 |
+
2026-06-06 10:15:42,456 INFO MainThread:430688 [wandb_run.py:_redirect():2305] redirect: wrap_raw
|
| 24 |
+
2026-06-06 10:15:42,456 INFO MainThread:430688 [wandb_run.py:_redirect():2370] Wrapping output streams.
|
| 25 |
+
2026-06-06 10:15:42,456 INFO MainThread:430688 [wandb_run.py:_redirect():2395] Redirects installed.
|
| 26 |
+
2026-06-06 10:15:42,458 INFO MainThread:430688 [wandb_init.py:init():911] run started, returning control to user process
|
| 27 |
+
2026-06-06 11:01:26,713 WARNING MsgRouterThr:430688 [router.py:message_loop():75] message_loop has been closed
|
run-20260606_101540-71wbmn3f/run-71wbmn3f.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f214aa38486f3f05d1eee0f14f5a0e8a435b8dfdde18051849b3c0a22c6a284b
|
| 3 |
+
size 5740137
|
run-20260606_110607-g0uqiw13/files/config.yaml
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.18.7
|
| 4 |
+
m: []
|
| 5 |
+
python_version: 3.10.20
|
| 6 |
+
t:
|
| 7 |
+
"1":
|
| 8 |
+
- 1
|
| 9 |
+
- 11
|
| 10 |
+
- 41
|
| 11 |
+
- 49
|
| 12 |
+
- 55
|
| 13 |
+
- 63
|
| 14 |
+
- 71
|
| 15 |
+
- 83
|
| 16 |
+
- 98
|
| 17 |
+
"2":
|
| 18 |
+
- 1
|
| 19 |
+
- 11
|
| 20 |
+
- 41
|
| 21 |
+
- 49
|
| 22 |
+
- 53
|
| 23 |
+
- 55
|
| 24 |
+
- 63
|
| 25 |
+
- 71
|
| 26 |
+
- 83
|
| 27 |
+
- 98
|
| 28 |
+
"3":
|
| 29 |
+
- 13
|
| 30 |
+
- 23
|
| 31 |
+
- 55
|
| 32 |
+
- 61
|
| 33 |
+
"4": 3.10.20
|
| 34 |
+
"5": 0.18.7
|
| 35 |
+
"6": 4.45.2
|
| 36 |
+
"8":
|
| 37 |
+
- 5
|
| 38 |
+
"12": 0.18.7
|
| 39 |
+
"13": linux-x86_64
|
run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_0cb1045c40045005886e.png
ADDED
|
run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_1f7b70d4340f1682d116.png
ADDED
|
Git LFS Details
|
run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_243c70f367b210f65092.png
ADDED
|
Git LFS Details
|
run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_3b28a4e69bd3270b76ad.png
ADDED
|
run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_43cdd7a37f85c6a13c2e.png
ADDED
|
Git LFS Details
|
run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_66feaebf5350fc850b80.png
ADDED
|
Git LFS Details
|
run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_6f4aa734b7c88a6d9df9.png
ADDED
|
Git LFS Details
|
run-20260606_110607-g0uqiw13/files/media/images/eval_images_0_78450a1ff8752a736686.png
ADDED
|
Git LFS Details
|